Fix benchmark flakiness

tracer/test/benchmarks/Benchmarks.Trace/AgentWriterBenchmark.cs

@@ -15,27 +15,18 @@ namespace Benchmarks.Trace
     [MemoryDiagnoser]
     [BenchmarkAgent1]
     [BenchmarkCategory(Constants.TracerCategory)]
-
     public class AgentWriterBenchmark
     {
         private const int SpanCount = 1000;
 
-        private static readonly IAgentWriter AgentWriter;
-        private static readonly ArraySegment<Span> EnrichedSpans;
-        static AgentWriterBenchmark()
-        {
-            var overrides = new NameValueConfigurationSource(new()
-            {
-                { ConfigurationKeys.StartupDiagnosticLogEnabled, false.ToString() },
-                { ConfigurationKeys.TraceEnabled, false.ToString() },
-            });
-            var sources = new CompositeConfigurationSource(new[] { overrides, GlobalConfigurationSource.Instance });
-            var settings = new TracerSettings(sources);
-
-            var api = new Api(new FakeApiRequestFactory(settings.Exporter.AgentUri), statsd: null, updateSampleRates: null, partialFlushEnabled: false);
-
-            AgentWriter = new AgentWriter(api, statsAggregator: null, statsd: null, automaticFlush: false);
+        private static IAgentWriter _agentWriter;
+        private static ArraySegment<Span> _enrichedSpans;
 
+        [GlobalSetup]
+        public void GlobalSetup()
+        {
+            // Create spans in GlobalSetup, not static constructor
+            // This ensures BenchmarkDotNet excludes allocation overhead from measurements
             var enrichedSpans = new Span[SpanCount];
             var now = DateTimeOffset.UtcNow;
 
@@ -46,10 +37,22 @@ static AgentWriterBenchmark()
                 enrichedSpans[i].SetMetric(Metrics.SamplingRuleDecision, 1.0);
             }
 
-            EnrichedSpans = new ArraySegment<Span>(enrichedSpans);
+            _enrichedSpans = new ArraySegment<Span>(enrichedSpans);
+
+            var overrides = new NameValueConfigurationSource(new()
+            {
+                { ConfigurationKeys.StartupDiagnosticLogEnabled, false.ToString() },
+                { ConfigurationKeys.TraceEnabled, false.ToString() },
+            });
+            var sources = new CompositeConfigurationSource(new[] { overrides, GlobalConfigurationSource.Instance });
+            var settings = new TracerSettings(sources);
+
+            var api = new Api(new FakeApiRequestFactory(settings.Exporter.AgentUri), statsd: null, updateSampleRates: null, partialFlushEnabled: false);
+
+            _agentWriter = new AgentWriter(api, statsAggregator: null, statsd: null, automaticFlush: false);
 
-            // Run benchmarks once to reduce noise
-            new AgentWriterBenchmark().WriteAndFlushEnrichedTraces().GetAwaiter().GetResult();
+            // Warmup to reduce noise
+            WriteAndFlushEnrichedTraces().GetAwaiter().GetResult();
         }
 
         /// <summary>
@@ -58,8 +61,8 @@ static AgentWriterBenchmark()
         [Benchmark]
         public Task WriteAndFlushEnrichedTraces()
         {
-            AgentWriter.WriteTrace(EnrichedSpans);
-            return AgentWriter.FlushTracesAsync();
+            _agentWriter.WriteTrace(_enrichedSpans);
+            return _agentWriter.FlushTracesAsync();
         }
 
         /// <summary>

tracer/test/benchmarks/Benchmarks.Trace/Asm/AppSecBodyBenchmark.cs

@@ -29,35 +29,23 @@ namespace Benchmarks.Trace.Asm
     [IgnoreProfile]
     public class AppSecBodyBenchmark
     {
-        private static readonly Security _security;
-        private readonly ComplexModel _complexModel = new()
-        {
-            Age = 12,
-            Gender = "Female",
-            Name = "Tata",
-            LastName = "Toto",
-            Address = new Address { Number = 12, City = new City { Name = "Paris", Country = new Country { Name = "France", Continent = new Continent { Name = "Europe", Planet = new Planet { Name = "Earth" } } } }, IsHouse = false, NameStreet = "lorem ipsum dolor sit amet" },
-            Address2 = new Address { Number = 15, City = new City { Name = "Madrid", Country = new Country { Name = "Spain", Continent = new Continent { Name = "Europe", Planet = new Planet { Name = "Earth" } } } }, IsHouse = true, NameStreet = "lorem ipsum dolor sit amet" },
-            Dogs = new List<Dog> { new Dog { Name = "toto", Dogs = new List<Dog> { new Dog { Name = "titi" }, new Dog { Name = "titi" } } }, new Dog { Name = "toto", Dogs = new List<Dog> { new Dog { Name = "tata" }, new Dog { Name = "tata" } } }, new Dog { Name = "tata", Dogs = new List<Dog> { new Dog { Name = "titi" }, new Dog { Name = "titi" }, new Dog { Name = "tutu" } } } }
-        };
-
-        private readonly Props10String _props10 = ConstructionUtils.ConstructProps10String();
-        private readonly Props100String _props100 = ConstructionUtils.ConstructProps100String();
-        private readonly Props1000String _props1000 = ConstructionUtils.ConstructProps1000String();
-
-        private readonly Props10Rec _props10x3 = ConstructionUtils.ConstructProps10Rec(3);
-        private readonly Props10Rec _props10x6 = ConstructionUtils.ConstructProps10Rec(6);
-
-
-
-        private static HttpContext _httpContext;
-
-        static AppSecBodyBenchmark()
+        private Security _security;
+        private ComplexModel _complexModel;
+        private Props10String _props10;
+        private Props100String _props100;
+        private Props1000String _props1000;
+        private Props10Rec _props10x3;
+        private Props10Rec _props10x6;
+        private HttpContext _httpContext;
+
+        [GlobalSetup]
+        public void GlobalSetup()
         {
             AppSecBenchmarkUtils.SetupDummyAgent();
             var dir = Directory.GetCurrentDirectory();
             Environment.SetEnvironmentVariable("DD_APPSEC_ENABLED", "true");
             _security = Security.Instance;
+
 #if NETFRAMEWORK
             var ms = new MemoryStream();
             using var sw = new StreamWriter(ms);
@@ -66,6 +54,27 @@ static AppSecBodyBenchmark()
 #else
             _httpContext = new DefaultHttpContext();
 #endif
+
+            _complexModel = new()
+            {
+                Age = 12,
+                Gender = "Female",
+                Name = "Tata",
+                LastName = "Toto",
+                Address = new Address { Number = 12, City = new City { Name = "Paris", Country = new Country { Name = "France", Continent = new Continent { Name = "Europe", Planet = new Planet { Name = "Earth" } } } }, IsHouse = false, NameStreet = "lorem ipsum dolor sit amet" },
+                Address2 = new Address { Number = 15, City = new City { Name = "Madrid", Country = new Country { Name = "Spain", Continent = new Continent { Name = "Europe", Planet = new Planet { Name = "Earth" } } } }, IsHouse = true, NameStreet = "lorem ipsum dolor sit amet" },
+                Dogs = new List<Dog> { new Dog { Name = "toto", Dogs = new List<Dog> { new Dog { Name = "titi" }, new Dog { Name = "titi" } } }, new Dog { Name = "toto", Dogs = new List<Dog> { new Dog { Name = "tata" }, new Dog { Name = "tata" } } }, new Dog { Name = "tata", Dogs = new List<Dog> { new Dog { Name = "titi" }, new Dog { Name = "titi" }, new Dog { Name = "tutu" } } } }
+            };
+
+            _props10 = ConstructionUtils.ConstructProps10String();
+            _props100 = ConstructionUtils.ConstructProps100String();
+            _props1000 = ConstructionUtils.ConstructProps1000String();
+            _props10x3 = ConstructionUtils.ConstructProps10Rec(3);
+            _props10x6 = ConstructionUtils.ConstructProps10Rec(6);
+
+            // Warmup
+            AllCycleSimpleBody();
+            AllCycleMoreComplexBody();
         }
 
         [Benchmark]

tracer/test/benchmarks/Benchmarks.Trace/Asm/AppSecEncodeBenchmark.cs

@@ -21,17 +21,25 @@ namespace Benchmarks.Trace.Asm;
 [BenchmarkCategory(Constants.AppSecCategory)]
 public class AppSecEncoderBenchmark
 {
-    private static readonly Encoder _encoder;
-    private static readonly EncoderLegacy _encoderLegacy;
-    private static readonly NestedMap _args;
+    private static Encoder _encoder;
+    private static EncoderLegacy _encoderLegacy;
+    private static NestedMap _args;
 
-    static AppSecEncoderBenchmark()
+    [GlobalSetup]
+    public void GlobalSetup()
     {
         AppSecBenchmarkUtils.SetupDummyAgent();
         _encoder = new Encoder();
         var wafLibraryInvoker = AppSecBenchmarkUtils.CreateWafLibraryInvoker();
         _encoderLegacy = new EncoderLegacy(wafLibraryInvoker);
+
+        // Create test data in GlobalSetup, not static initializer
+        // This ensures BenchmarkDotNet excludes allocation overhead from measurements
         _args = MakeNestedMap(20);
+
+        // Warmup
+        EncodeArgs();
+        EncodeLegacyArgs();
     }
 
     /// <summary>

tracer/test/benchmarks/Benchmarks.Trace/Asm/AppSecWafBenchmark.cs

@@ -23,18 +23,21 @@ namespace Benchmarks.Trace.Asm;
 [BenchmarkAgent7]
 [BenchmarkCategory(Constants.AppSecCategory)]
 [IgnoreProfile]
+#if NETCOREAPP3_1
+[DisableTieredCompilation]
+#endif
 public class AppSecWafBenchmark
 {
     private const int TimeoutMicroSeconds = 1_000_000;
 
-    private static readonly Waf Waf;
+    private static Waf _waf;
+    private static Dictionary<string, object> _stage1;
+    private static Dictionary<string, object> _stage1Attack;
+    private static Dictionary<string, object> _stage2;
+    private static Dictionary<string, object> _stage3;
 
-    private static readonly Dictionary<string, object> stage1 = MakeRealisticNestedMapStage1(false);
-    private static readonly Dictionary<string, object> stage1Attack = MakeRealisticNestedMapStage1(true);
-    private static readonly Dictionary<string, object> stage2 = MakeRealisticNestedMapStage2();
-    private static readonly Dictionary<string, object> stage3 = MakeRealisticNestedMapStage3();
-
-    static AppSecWafBenchmark()
+    [GlobalSetup]
+    public void GlobalSetup()
     {
         AppSecBenchmarkUtils.SetupDummyAgent();
         var wafLibraryInvoker = AppSecBenchmarkUtils.CreateWafLibraryInvoker();
@@ -52,7 +55,32 @@ static AppSecWafBenchmark()
         {
             throw new ArgumentException($"Waf could not initialize, error message is: {initResult.ErrorMessage}");
         }
-        Waf = initResult.Waf;
+        _waf = initResult.Waf;
+
+        // Create test data in GlobalSetup, not static initializer
+        // This ensures BenchmarkDotNet excludes allocation overhead from measurements
+        _stage1 = MakeRealisticNestedMapStage1(false);
+        _stage1Attack = MakeRealisticNestedMapStage1(true);
+        _stage2 = MakeRealisticNestedMapStage2();
+        _stage3 = MakeRealisticNestedMapStage3();
+
+        // More aggressive warmup for native code paths (WAF library)
+        // Ensures JIT compilation completes and native context creation stabilizes
+        for (int i = 0; i < 10; i++)
+        {
+            RunWafRealisticBenchmark();
+            RunWafRealisticBenchmarkWithAttack();
+        }
+    }
+
+    [IterationSetup]
+    public void IterationSetup()
+    {
+        // Force GC to reduce variance from native memory interactions
+        // WAF library uses unmanaged memory with allocation patterns outside .NET GC control
+        GC.Collect();
+        GC.WaitForPendingFinalizers();
+        GC.Collect();
     }
 
     private static Dictionary<string, object> MakeRealisticNestedMapStage1(bool withAttack)
@@ -132,18 +160,18 @@ private static Dictionary<string, object> MakeRealisticNestedMapStage3()
     [Benchmark]
     public void RunWafRealisticBenchmark()
     {
-        var context = Waf.CreateContext();
-        context!.Run(stage1, TimeoutMicroSeconds);
-        context!.Run(stage2, TimeoutMicroSeconds);
-        context!.Run(stage3, TimeoutMicroSeconds);
+        var context = _waf.CreateContext();
+        context!.Run(_stage1, TimeoutMicroSeconds);
+        context!.Run(_stage2, TimeoutMicroSeconds);
+        context!.Run(_stage3, TimeoutMicroSeconds);
         context.Dispose();
     }
 
     [Benchmark]
     public void RunWafRealisticBenchmarkWithAttack()
     {
-        var context = Waf.CreateContext();
-        context!.Run(stage1Attack, TimeoutMicroSeconds);
+        var context = _waf.CreateContext();
+        context!.Run(_stage1Attack, TimeoutMicroSeconds);
         context.Dispose();
     }
 }

tracer/test/benchmarks/Benchmarks.Trace/AspNetCoreBenchmark.cs

@@ -21,9 +21,10 @@ namespace Benchmarks.Trace
     [BenchmarkCategory(Constants.TracerCategory)]
     public class AspNetCoreBenchmark
     {
-        private static readonly HttpClient Client;
+        private HttpClient _client;
 
-        static AspNetCoreBenchmark()
+        [GlobalSetup]
+        public void GlobalSetup()
         {
             var settings = TracerSettings.Create(new() { { ConfigurationKeys.StartupDiagnosticLogEnabled, false } });
 
@@ -33,18 +34,18 @@ static AspNetCoreBenchmark()
                 .UseStartup<Startup>();
 
             var testServer = new TestServer(builder);
-            Client = testServer.CreateClient();
+            _client = testServer.CreateClient();
 
             Datadog.Trace.ClrProfiler.Instrumentation.Initialize();
 
-            var bench = new AspNetCoreBenchmark();
-            bench.SendRequest();
+            // Warmup to initialize middleware pipeline
+            SendRequest();
         }
 
         [Benchmark]
         public string SendRequest()
         {
-            return Client.GetStringAsync("/Home").GetAwaiter().GetResult();
+            return _client.GetStringAsync("/Home").GetAwaiter().GetResult();
         }
 
         private class Startup
@@ -96,6 +97,11 @@ namespace Benchmarks.Trace
     [MemoryDiagnoser]
     public class AspNetCoreBenchmark
     {
+        [GlobalSetup]
+        public void GlobalSetup()
+        {
+        }
+
         [Benchmark]
         public string SendRequest()
         {

tracer/test/benchmarks/Benchmarks.Trace/CIVisibilityProtocolWriterBenchmark.cs

@@ -17,21 +17,14 @@ public class CIVisibilityProtocolWriterBenchmark
     {
         private const int SpanCount = 1000;
 
-        private static readonly IEventWriter EventWriter;
-        private static readonly ArraySegment<Span> EnrichedSpans;
+        private static IEventWriter _eventWriter;
+        private static ArraySegment<Span> _enrichedSpans;
 
-        static CIVisibilityProtocolWriterBenchmark()
+        [GlobalSetup]
+        public void GlobalSetup()
         {
-            var overrides = new NameValueConfigurationSource(new()
-            {
-                { ConfigurationKeys.StartupDiagnosticLogEnabled, false.ToString() },
-                { ConfigurationKeys.TraceEnabled, false.ToString() },
-            });
-            var sources = new CompositeConfigurationSource(new[] { overrides, GlobalConfigurationSource.Instance });
-            var settings = new TestOptimizationSettings(sources, NullConfigurationTelemetry.Instance);
-
-            EventWriter = new CIVisibilityProtocolWriter(settings, new FakeCIVisibilityProtocolWriter());
-
+            // Create spans in GlobalSetup, not static constructor
+            // This ensures BenchmarkDotNet excludes allocation overhead from measurements
             var enrichedSpans = new Span[SpanCount];
             var now = DateTimeOffset.UtcNow;
             for (var i = 0; i < SpanCount; i++)
@@ -41,10 +34,20 @@ static CIVisibilityProtocolWriterBenchmark()
                 enrichedSpans[i].SetMetric(Metrics.SamplingRuleDecision, 1.0);
             }
 
-            EnrichedSpans = new ArraySegment<Span>(enrichedSpans);
+            _enrichedSpans = new ArraySegment<Span>(enrichedSpans);
+
+            var overrides = new NameValueConfigurationSource(new()
+            {
+                { ConfigurationKeys.StartupDiagnosticLogEnabled, false.ToString() },
+                { ConfigurationKeys.TraceEnabled, false.ToString() },
+            });
+            var sources = new CompositeConfigurationSource(new[] { overrides, GlobalConfigurationSource.Instance });
+            var settings = new TestOptimizationSettings(sources, NullConfigurationTelemetry.Instance);
+
+            _eventWriter = new CIVisibilityProtocolWriter(settings, new FakeCIVisibilityProtocolWriter());
 
-            // Run benchmarks once to reduce noise
-            new CIVisibilityProtocolWriterBenchmark().WriteAndFlushEnrichedTraces().GetAwaiter().GetResult();
+            // Warmup to reduce noise
+            WriteAndFlushEnrichedTraces().GetAwaiter().GetResult();
         }
 
         /// <summary>
@@ -53,8 +56,8 @@ static CIVisibilityProtocolWriterBenchmark()
         [Benchmark]
         public Task WriteAndFlushEnrichedTraces()
         {
-            EventWriter.WriteTrace(EnrichedSpans);
-            return EventWriter.FlushTracesAsync();
+            _eventWriter.WriteTrace(_enrichedSpans);
+            return _eventWriter.FlushTracesAsync();
         }
 
         private class FakeCIVisibilityProtocolWriter : ICIVisibilityProtocolWriterSender

tracer/test/benchmarks/Benchmarks.Trace/CharSliceBenchmark.cs

@@ -13,8 +13,20 @@ namespace Benchmarks.Trace;
 [MemoryDiagnoser]
 [BenchmarkAgent1]
 [BenchmarkCategory(Constants.TracerCategory)]
+#if NETCOREAPP3_1
+[DisableTieredCompilation]
+#endif
 public class CharSliceBenchmark
 {
+    [IterationSetup]
+    public void Setup()
+    {
+        // Force GC to ensure clean state and reduce variance from GC timing
+        GC.Collect();
+        GC.WaitForPendingFinalizers();
+        GC.Collect();
+    }
+
     [Benchmark]
     public void OriginalCharSlice()
     {