chore(diagnostics): update netcore example with activity correlation

Update NetCoreSample with an OpenTelemetry based listener to enable
the Hangfire activity source.

Add initial activity creation along with logging of TraceId to show
job correlation in log output. Add background job examples, including
error examples.

Author: sgryphon

NuGet.config

@@ -12,7 +12,7 @@
 
   <trustedSigners> 
     <repository name="nuget.org" serviceIndex="https://api.nuget.org/v3/index.json">
-      <owners>Microsoft;aspnet;dotnetframework;HangfireIO;xunit;jamesnk;kzu;castleproject;psake;ILRepack;davidebbo;StackExchange;Dapper;brady.holt;dwhelan;raboof;damianh;</owners>
+      <owners>Microsoft;aspnet;dotnetframework;HangfireIO;xunit;jamesnk;kzu;castleproject;psake;ILRepack;davidebbo;StackExchange;Dapper;brady.holt;dwhelan;raboof;damianh;OpenTelemetry;</owners>
       <certificate fingerprint="5a2901d6ada3d18260b9c6dfe2133c95d74b9eef6ae0e5dc334c8454d1477df4" hashAlgorithm="SHA256" allowUntrustedRoot="false" />
       <certificate fingerprint="0e5f38f57dc1bcc806d8494f4f90fbcedd988b46760709cbeec6f4219aa6157d" hashAlgorithm="SHA256" allowUntrustedRoot="false" />
       <certificate fingerprint="1f4b311d9acc115c8dc8018b5a49e00fce6da8e2855f9f014ca6f34570bc482d" hashAlgorithm="SHA256" allowUntrustedRoot="false" /> 

samples/NetCoreSample/NetCoreSample.csproj

@@ -16,6 +16,9 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0" />
+    <PackageReference Include="Hangfire.InMemory" Version="1.0.0" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.9.0" />
+    <PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.9.0" />
   </ItemGroup>
 
 </Project>

samples/NetCoreSample/Program.cs

@@ -1,27 +1,43 @@
 using System;
 using System.Data;
+using System.Diagnostics;
 using System.Threading;
 using System.Threading.Tasks;
 using Hangfire;
 using Hangfire.Annotations;
 using Hangfire.Client;
 using Hangfire.Common;
+using Hangfire.Diagnostics;
+using Hangfire.InMemory;
 using Hangfire.Server;
 using Hangfire.SqlServer;
 using Hangfire.States;
+using Microsoft.AspNetCore.Mvc.Infrastructure;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
+using OpenTelemetry.Trace;
 
 namespace NetCoreSample
 {
     class Program
     {
+        // To use in-memory store instead of database:
+        //   dotnet run -- --UseInMemory true
+
+        // To show trace console exporter output: 
+        //   dotnet run -- --TraceConsoleExporter true
+
+        public static readonly ActivitySource ActivitySource = new ActivitySource(nameof(NetCoreSample));
+
         static async Task Main(string[] args)
         {
-            var host = new HostBuilder()
-                .ConfigureLogging(x => x.AddConsole().SetMinimumLevel(LogLevel.Information))
+            var host = Host.CreateDefaultBuilder(args)
+                .ConfigureLogging(x => x
+                    .AddSimpleConsole()
+                    .SetMinimumLevel(LogLevel.Information))
                 .ConfigureServices((hostContext, services) =>
                 {
                     services.Configure<HostOptions>(option =>
@@ -49,19 +65,40 @@ static async Task Main(string[] args)
                     services.TryAddSingleton<IBackgroundJobStateChanger>(x => new CustomBackgroundJobStateChanger(
                             new BackgroundJobStateChanger(x.GetRequiredService<IJobFilterProvider>())));
 
-                    services.AddHangfire((provider, configuration) => configuration
+                    var useInMemory = hostContext.Configuration.GetValue<bool>("UseInMemory");
+                    services.AddHangfire((provider, configuration) => {
+                        configuration
                         .SetDataCompatibilityLevel(CompatibilityLevel.Version_170)
-                        .UseSimpleAssemblyNameTypeSerializer()
-                        .UseSqlServerStorage(
-                            @"Server=.\;Database=Hangfire.Sample;Trusted_Connection=True;", 
-                            provider.GetRequiredService<SqlServerStorageOptions>()));
+                        .UseSimpleAssemblyNameTypeSerializer();
+                        if (useInMemory) {
+                            configuration.UseInMemoryStorage();
+                        }
+                        else
+                        {
+                            configuration.UseSqlServerStorage(
+                                @"Server=.\;Database=Hangfire.Sample;Trusted_Connection=True;", 
+                                provider.GetRequiredService<SqlServerStorageOptions>());
+                        }
+                    });
 
                     services.AddHostedService<RecurringJobsService>();
+                    services.AddHostedService<BackgroundJobsService>();
                     services.AddHangfireServer(options =>
                     {
                         options.StopTimeout = TimeSpan.FromSeconds(15);
                         options.ShutdownTimeout = TimeSpan.FromSeconds(30);
                     });
+
+                    var traceConsoleExporter = hostContext.Configuration.GetValue<bool>("TraceConsoleExporter");
+                    services.AddOpenTelemetry()
+                        .WithTracing(tracing => {
+                            tracing.AddSource(DiagnosticHeaders.DefaultListenerName);
+                            tracing.AddSource(nameof(NetCoreSample));
+                            if (traceConsoleExporter)
+                            {
+                                tracing.AddConsoleExporter();
+                            }
+                        });
                 })
                 .Build();
 
@@ -123,24 +160,35 @@ internal class RecurringJobsService : BackgroundService
     {
         private readonly IBackgroundJobClient _backgroundJobs;
         private readonly IRecurringJobManager _recurringJobs;
-        private readonly ILogger<RecurringJobScheduler> _logger;
+        private readonly ILogger<RecurringJobsService> _logger;
+        private readonly ILoggerFactory _loggerFactory;
 
         public RecurringJobsService(
             [NotNull] IBackgroundJobClient backgroundJobs,
             [NotNull] IRecurringJobManager recurringJobs,
-            [NotNull] ILogger<RecurringJobScheduler> logger)
+            [NotNull] ILogger<RecurringJobsService> logger,
+            ILoggerFactory loggerFactory)
         {
             _backgroundJobs = backgroundJobs ?? throw new ArgumentNullException(nameof(backgroundJobs));
             _recurringJobs = recurringJobs ?? throw new ArgumentNullException(nameof(recurringJobs));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+            _loggerFactory = loggerFactory;
         }
 
         protected override Task ExecuteAsync(CancellationToken stoppingToken)
         {
             try
             {
-                _recurringJobs.AddOrUpdate("seconds", () => Console.WriteLine("Hello, seconds!"), "*/15 * * * * *");
-                _recurringJobs.AddOrUpdate("minutely", () => Console.WriteLine("Hello, world!"), Cron.Minutely);
+                _logger.LogInformation("Creating recurring jobs");
+
+                _recurringJobs.AddOrUpdate("seconds", () => Hello("seconds"), "*/15 * * * * *");
+
+                using (var activity =  Program.ActivitySource.StartActivity("enqueue"))
+                {
+                    _logger.LogInformation("Creating job minutely (hello world), trace_id={ActivityTraceId}", activity.TraceId);
+                    _recurringJobs.AddOrUpdate("minutely", () => Hello("world"), Cron.Minutely);
+                }
+
                 _recurringJobs.AddOrUpdate("hourly", () => Console.WriteLine("Hello"), "25 15 * * *");
                 _recurringJobs.AddOrUpdate("neverfires", () => Console.WriteLine("Can only be triggered"), "0 0 31 2 *");
 
@@ -161,5 +209,71 @@ protected override Task ExecuteAsync(CancellationToken stoppingToken)
 
             return Task.CompletedTask;
         }
+
+        public void Hello(string name)
+        {
+            Console.WriteLine($"Hello, {name}!");
+            var logger = _loggerFactory.CreateLogger<RecurringJobsService>();
+            logger.LogInformation("Hello, {Name}! trace_id={ActivityTraceId}", name, Activity.Current?.TraceId);
+        }
+    }
+
+    internal class BackgroundJobsService : BackgroundService
+    {
+        private readonly IBackgroundJobClient _backgroundJobs;
+        private readonly ILogger _logger;
+        private readonly ILoggerFactory _loggerFactory;
+
+        public BackgroundJobsService(
+            [NotNull] IBackgroundJobClient backgroundJobs,
+            [NotNull] ILogger<BackgroundJobsService> logger,
+            ILoggerFactory loggerFactory)
+        {
+            _backgroundJobs = backgroundJobs ?? throw new ArgumentNullException(nameof(backgroundJobs));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+            _loggerFactory = loggerFactory;
+        }
+
+        protected override Task ExecuteAsync(CancellationToken stoppingToken)
+        {
+            try
+            {
+                _logger.LogInformation("Creating backgriound jobs");
+
+                using (var activity =  Program.ActivitySource.StartActivity("enqueue"))
+                {
+                    _logger.LogInformation("Creating job 10, trace_id={ActivityTraceId}", activity.TraceId);
+                    var jobId1 = _backgroundJobs.Enqueue(() => Job(10));
+                }
+                using (var activity =  Program.ActivitySource.StartActivity("schedule"))
+                {
+                    _logger.LogInformation("Scheduling job 20, continue with 30, trace_id={ActivityTraceId}", activity.TraceId);
+                    var jobId2 = _backgroundJobs.Schedule(() => Job(20), TimeSpan.FromSeconds(30));
+                    var jobId3 = _backgroundJobs.ContinueJobWith(jobId2, () => Job(30));
+                }
+                using (var activity =  Program.ActivitySource.StartActivity("error"))
+                {
+                    _logger.LogInformation("Scheduling error job 40, trace_id={ActivityTraceId}", activity.TraceId);
+                    var jobId4 = _backgroundJobs.Schedule(() => Job(40), TimeSpan.FromSeconds(60));
+                }
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "An exception occurred while creating recurring jobs.");
+            }
+
+            return Task.CompletedTask;
+        }
+
+        public void Job(int counter) {
+            Console.WriteLine("Hello, job {0}!", counter);
+            var logger = _loggerFactory.CreateLogger<BackgroundJobsService>();
+            logger.LogInformation("Hello, job {Counter} trace_id={ActivityTraceId}", counter, Activity.Current?.TraceId);
+            if (counter == 40)
+            {
+                throw new InvalidOperationException("Counter 40 is invalid.");
+            }
+        }
     }
+
 }