Instrument .NET Azure WebJobs for Splunk Observability Cloud 🔗

You can instrument applications or services running on Azure WebJobs using the OpenTelemetry .NET SDK. Follow these instructions to get started.

Define the environment variables 🔗

Set the required environment variables for your application.

Select your application in Azure Web App.
Go to Settings, Configuration.

Select New application setting to add the following settings:

Name	Value
`SPLUNK_ACCESS_TOKEN`	Your Splunk access token. To obtain an access token, see Retrieve and manage user API access tokens using Splunk Observability Cloud.
`SPLUNK_REALM`	Your Splunk Observability Cloud realm, for example `us0`. To find your realm, see Note about realms.

Add any other settings you might need. See Configure the Splunk Distribution of OpenTelemetry .NET.

Add the required libraries using NuGet 🔗

Add the following libraries using the NuGet package manager in Visual Studio.

Isolated worker process function 🔗

Activate the Include prerelease setting.
Install the latest version of the following libraries:

Initialize OpenTelemetry in the code 🔗

After adding the dependencies, create an OpenTelemetry helper for your application:

using Microsoft.Azure.WebJobs;
using Microsoft.Azure.WebJobs.Host;
using Microsoft.Extensions.DependencyInjection;
using OpenTelemetry.Exporter;
using OpenTelemetry.Logs;
using OpenTelemetry.Metrics;
using OpenTelemetry.ResourceDetectors.Azure;
using OpenTelemetry.Resources;
using OpenTelemetry.Trace;
using System.Diagnostics;

namespace <YourNamespaceHere>.Helpers;

internal static class SplunkOpenTelemetry
{
    private static readonly string AccessToken;
    private static readonly string Realm;

    static SplunkOpenTelemetry()
    {
        AccessToken = Environment.GetEnvironmentVariable("SPLUNK_ACCESS_TOKEN")?.Trim()
            ?? throw new ArgumentNullException("SPLUNK_ACCESS_TOKEN");

        Realm = Environment.GetEnvironmentVariable("SPLUNK_REALM")?.Trim()
            ?? throw new ArgumentNullException("SPLUNK_REALM");
    }

    public static IWebJobsBuilder AddSplunkOpenTelemetry(this IWebJobsBuilder builder)
    {
        // Get environment variables from function configuration
        // You need a valid Splunk Observability Cloud access token and realm
        var serviceName = Environment.GetEnvironmentVariable("WEBSITE_SITE_NAME") ?? "Unknown";
        var enableTraceResponseHeaderValue = Environment.GetEnvironmentVariable("SPLUNK_TRACE_RESPONSE_HEADER_ENABLED")?.Trim();

        // See https://github.com/open-telemetry/opentelemetry-dotnet-contrib/tree/main/src/OpenTelemetry.ResourceDetectors.Azure
        // for other types of Azure detectors
        var resourceDetector = new AppServiceResourceDetector();

        builder.Services.AddOpenTelemetry()
            .WithTracing(t => t
                // Use Add[instrumentation-name]Instrumentation to instrument missing services
                // Use Nuget to find different instrumentation libraries
                .AddHttpClientInstrumentation(opts =>
                {
                    // This filter prevents background (parent-less) http client activity
                    opts.FilterHttpWebRequest = req => Activity.Current?.Parent != null;
                    opts.FilterHttpRequestMessage = req => Activity.Current?.Parent != null;
                })
                // Use AddSource to add your custom DiagnosticSource source names
                //.AddSource("My.Source.Name")
                // Automatically creates the root span with function start
                .AddSource(SplunkFunctionAttribute.ActivitySourceName)
                .SetSampler(new AlwaysOnSampler())
                .ConfigureResource(cfg => cfg
                    .AddService(serviceName: serviceName, serviceVersion: "1.0.0")
                    .AddDetector(resourceDetector))
                .AddConsoleExporter()
                .AddOtlpExporter(opts =>
                {
                    opts.Endpoint = new Uri($"https://ingest.{Realm}.signalfx.com/v2/trace/otlp");
                    opts.Protocol = OtlpExportProtocol.HttpProtobuf;
                    opts.Headers = $"X-SF-TOKEN={AccessToken}";
                }))
            .WithMetrics(m => m
                // Use Add[instrumentation-name]Instrumentation to instrument missing services
                // Use Nuget to find different instrumentation libraries
                .AddHttpClientInstrumentation()
                .AddRuntimeInstrumentation()
                .AddProcessInstrumentation()
                .ConfigureResource(cfg => cfg
                    .AddService(serviceName: serviceName, serviceVersion: "1.0.0")
                    .AddDetector(resourceDetector))
                .AddOtlpExporter(opts =>
                {
                    opts.Endpoint = new Uri($"https://ingest.{Realm}.signalfx.com/v2/datapoint/otlp");
                    opts.Headers = $"X-SF-TOKEN={AccessToken}";
                }));

        return builder;
    }
}

internal class SplunkFunctionAttribute : FunctionInvocationFilterAttribute
{
    public const string ActivitySourceName = "Splunk.Azure.WebJob";

    private static readonly ActivitySource ActivitySource = new(ActivitySourceName);

    private Activity? _currentActivity;

    public override Task OnExecutingAsync(FunctionExecutingContext executingContext, CancellationToken cancellationToken)
    {
        _currentActivity = ActivitySource.StartActivity(executingContext.FunctionName, ActivityKind.Server);
        _currentActivity?.AddTag("faas.name", executingContext.FunctionName);
        _currentActivity?.AddTag("faas.instance", executingContext.FunctionInstanceId);

        return base.OnExecutingAsync(executingContext, cancellationToken);
    }

    public override Task OnExecutedAsync(FunctionExecutedContext executedContext, CancellationToken cancellationToken)
    {
        if (!executedContext.FunctionResult.Succeeded)
        {
            if (executedContext.FunctionResult.Exception != null)
            {
                _currentActivity?.SetStatus(Status.Error.WithDescription(executedContext.FunctionResult.Exception.Message));
                _currentActivity?.RecordException(executedContext.FunctionResult.Exception);
            }
            else
            {
                _currentActivity?.SetStatus(Status.Error);
            }
        }

        _currentActivity?.Stop();

        return base.OnExecutedAsync(executedContext, cancellationToken);
    }
}

Use the helper you created in the Program.cs file:

var builder = new HostBuilder()
.ConfigureWebJobs(context =>
{
    context.AddSplunkOpenTelemetry();
})

Use the SplunkFunctionAttribute with every defined WebJobs Function. See the example how to properly attribute your function.

public class Functions
{
    [SplunkFunction]
    public void ProcessTimer([TimerTrigger("1/5 * * * * *")] TimerInfo timerInfo, ILogger logger)
    {
        logger.LogInformation("Hello World!");
    }
}

Check that data is coming in 🔗

Run your function and search for spans in Splunk APM. See View and filter for spans within a trace for more information.

Troubleshooting 🔗

If you are a Splunk Observability Cloud customer and are not able to see your data in Splunk Observability Cloud, you can get help in the following ways.

Available to Splunk Observability Cloud customers

Submit a case in the Splunk Support Portal .
Call Splunk Customer Support .

Available to prospective customers and free trial users

Ask a question and get answers through community support at Splunk Answers .
Join the Splunk #observability user group Slack channel to communicate with customers, partners, and Splunk employees worldwide. To join, see Chat groups in the Get Started with Splunk Community manual.

To learn about even more support options, see Splunk Customer Success .

Related Topics