From 0e9d6974e0a6cc060d503f2ce1e7c14463c9ff92 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Tue, 23 Sep 2025 18:14:45 +0100 Subject: [PATCH 1/9] Create a SettingsManager for tracking and notifying about settings changes --- .../Configuration/SettingsManager.cs | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 tracer/src/Datadog.Trace/Configuration/SettingsManager.cs diff --git a/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs b/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs new file mode 100644 index 000000000000..1cebc3a3d2d5 --- /dev/null +++ b/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs @@ -0,0 +1,205 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Threading; +using Datadog.Trace.Configuration.ConfigurationSources; +using Datadog.Trace.Configuration.ConfigurationSources.Telemetry; +using Datadog.Trace.Configuration.Telemetry; +using Datadog.Trace.Logging; + +namespace Datadog.Trace.Configuration; + +internal class SettingsManager( + TracerSettings tracerSettings, MutableSettings initialMutable, ExporterSettings initialExporter) +{ + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); + private readonly TracerSettings _tracerSettings = tracerSettings; + private readonly List _subscribers = []; + private SettingChanges? _latest; + + /// + /// Gets the initial . On app startup, these will be the values read from + /// static sources. To subscribe to updates to these settings, from code or remote config, call . + /// + public MutableSettings InitialMutableSettings { get; } = initialMutable; + + /// + /// Gets the initial . On app startup, these will be the values read from + /// static sources. To subscribe to updates to these settings, from code or remote config, call . + /// + public ExporterSettings InitialExporterSettings { get; } = initialExporter; + + /// + /// Subscribe to changes in and/or . + /// Called whenever these settings change. If the settings have already changed when + /// is called, is invoked immediately with the latest configuration. + /// Also note that calling twice with the same callback + /// will invoke the callback twice. + /// + /// The method to invoke + /// An that should be disposed to unsubscribe + public IDisposable SubscribeToChanges(Action callback) + { + var subscription = new SettingChangeSubscription(this, callback); + lock (_subscribers) + { + _subscribers.Add(subscription); + } + + if (Volatile.Read(ref _latest) is { } currentConfig) + { + try + { + // If we already have updates, call this immediately + callback(currentConfig); + } + catch (Exception ex) + { + Log.Error(ex, "Error notifying subscriber of updated MutableSettings during subscribe"); + } + } + + return subscription; + } + + /// + /// Regenerate the application's new based on runtime configuration sources + /// + /// An for dynamic config via remote config + /// An for manual configuration (in code) + /// The central to report config telemetry updates to + /// True if changes were detected and consumers were updated, false otherwise + public bool UpdateSettings( + IConfigurationSource dynamicConfigSource, + ManualInstrumentationConfigurationSourceBase manualSource, + IConfigurationTelemetry centralTelemetry) + { + if (BuildNewSettings(dynamicConfigSource, manualSource, centralTelemetry) is { } newSettings) + { + NotifySubscribers(newSettings); + return true; + } + + return false; + } + + // Internal for testing + internal SettingChanges? BuildNewSettings( + IConfigurationSource dynamicConfigSource, + ManualInstrumentationConfigurationSourceBase manualSource, + IConfigurationTelemetry centralTelemetry) + { + var initialSettings = manualSource.UseDefaultSources + ? InitialMutableSettings + : MutableSettings.CreateWithoutDefaultSources(_tracerSettings); + + var current = Volatile.Read(ref _latest); + var currentMutable = current?.Mutable ?? InitialMutableSettings; + var currentExporter = current?.Exporter ?? InitialExporterSettings; + + var telemetry = new ConfigurationTelemetry(); + var newMutableSettings = MutableSettings.CreateUpdatedMutableSettings( + dynamicConfigSource, + manualSource, + initialSettings, + _tracerSettings, + telemetry, + new OverrideErrorLog()); // TODO: We'll later report these + + // The only exporter setting we currently _allow_ to change is the AgentUri, but if that does change, + // it can mean that _everything_ about the exporter settings changes. To minimize the work to do, and + // to simplify comparisons, we try to read the agent url from the manual setting. If it's missing, not + // set, or unchanged, there's no need to update the exporter settings. + // We only technically need to do this today if _manual_ config changes, not if remote config changes, + // but for simplicity we don't distinguish currently. + var exporterTelemetry = new ConfigurationTelemetry(); + var newRawExporterSettings = ExporterSettings.Raw.CreateUpdatedFromManualConfig( + currentExporter.RawSettings, + manualSource, + exporterTelemetry, + manualSource.UseDefaultSources); + + var isSameMutableSettings = currentMutable.Equals(newMutableSettings); + var isSameExporterSettings = currentExporter.RawSettings.Equals(newRawExporterSettings); + + if (isSameMutableSettings && isSameExporterSettings) + { + Log.Debug("No changes detected in the new configuration"); + // Even though there were no "real" changes, there may be _effective_ changes in telemetry that + // need to be recorded (e.g. the customer set the value in code, but it was already set via + // env vars). We _should_ record exporter settings too, but that introduces a bunch of complexity + // which we'll resolve later anyway, so just have that gap for now (it's very niche). + // If there are changes, they're recorded automatically in ConfigureInternal + telemetry.CopyTo(centralTelemetry); + return null; + } + + Log.Information("Notifying consumers of new settings"); + var updatedMutableSettings = isSameMutableSettings ? null : newMutableSettings; + var updatedExporterSettings = isSameExporterSettings ? null : new ExporterSettings(newRawExporterSettings, exporterTelemetry); + + return new SettingChanges(updatedMutableSettings, updatedExporterSettings); + } + + private void NotifySubscribers(SettingChanges settings) + { + // Strictly, for safety, we only need to lock in the subscribers list access. However, + // there's nothing to prevent NotifySubscribers being called concurrently, + // which could result in weird out-of-order notifications for customers. So for simplicity + // we just lock the whole method to ensure serialized updates. + + lock (_subscribers) + { + var subscribers = _subscribers; + Volatile.Write(ref _latest, settings); + + foreach (var subscriber in subscribers) + { + try + { + subscriber.Notify(settings); + } + catch (Exception ex) + { + Log.Error(ex, "Error notifying subscriber of MutableSettings change"); + } + } + } + } + + private sealed class SettingChangeSubscription(SettingsManager owner, Action notify) : IDisposable + { + private readonly SettingsManager _owner = owner; + + public Action Notify { get; } = notify; + + public void Dispose() + { + lock (_owner._subscribers) + { + _owner._subscribers.Remove(this); + } + } + } + + public sealed class SettingChanges(MutableSettings? mutableSettings, ExporterSettings? exporterSettings) + { + /// + /// Gets the new , if they have changed. + /// If there are no changes, returns null. + /// + public MutableSettings? Mutable { get; } = mutableSettings; + + /// + /// Gets the new , if they have changed. + /// If there are no changes, returns null. + /// + public ExporterSettings? Exporter { get; } = exporterSettings; + } +} From e822c6f704cd0a1d5ac0c1d4602fbaf98db3b507 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 22 Oct 2025 09:53:16 +0100 Subject: [PATCH 2/9] Store the SettingsManager on the global TracerManager instance --- .../Ci/TestOptimizationTracerManager.cs | 10 +++++++--- .../Ci/TestOptimizationTracerManagerFactory.cs | 7 ++++--- tracer/src/Datadog.Trace/Tracer.cs | 2 +- tracer/src/Datadog.Trace/TracerManager.cs | 4 ++++ .../src/Datadog.Trace/TracerManagerFactory.cs | 17 ++++++++++++----- .../TracerManagerFactoryTests.cs | 3 ++- 6 files changed, 30 insertions(+), 13 deletions(-) diff --git a/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManager.cs b/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManager.cs index f7f894912998..6899c7579c72 100644 --- a/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManager.cs +++ b/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManager.cs @@ -44,7 +44,8 @@ public TestOptimizationTracerManager( IRemoteConfigurationManager remoteConfigurationManager, IDynamicConfigurationManager dynamicConfigurationManager, ITracerFlareManager tracerFlareManager, - ISpanEventsManager spanEventsManager) + ISpanEventsManager spanEventsManager, + SettingsManager settingsManager) : base( settings, agentWriter, @@ -62,6 +63,7 @@ public TestOptimizationTracerManager( dynamicConfigurationManager, tracerFlareManager, spanEventsManager, + settingsManager, GetProcessors(settings.PartialFlushEnabled, agentWriter is CIVisibilityProtocolWriter)) { } @@ -160,7 +162,8 @@ public LockedManager( IRemoteConfigurationManager remoteConfigurationManager, IDynamicConfigurationManager dynamicConfigurationManager, ITracerFlareManager tracerFlareManager, - ISpanEventsManager spanEventsManager) + ISpanEventsManager spanEventsManager, + SettingsManager settingsManager) : base( settings, agentWriter, @@ -177,7 +180,8 @@ public LockedManager( remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager, - spanEventsManager) + spanEventsManager, + settingsManager) { } } diff --git a/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagerFactory.cs b/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagerFactory.cs index 53798be38d6f..f477c326cffc 100644 --- a/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagerFactory.cs @@ -53,15 +53,16 @@ protected override TracerManager CreateTracerManagerFrom( IRemoteConfigurationManager remoteConfigurationManager, IDynamicConfigurationManager dynamicConfigurationManager, ITracerFlareManager tracerFlareManager, - ISpanEventsManager spanEventsManager) + ISpanEventsManager spanEventsManager, + SettingsManager settingsManager) { telemetry.RecordTestOptimizationSettings(_settings); if (_testOptimizationTracerManagement.UseLockedTracerManager) { - return new TestOptimizationTracerManager.LockedManager(settings, agentWriter, scopeManager, statsd, runtimeMetrics, logSubmissionManager, telemetry, discoveryService, dataStreamsManager, gitMetadataTagsProvider, traceSampler, spanSampler, remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager, spanEventsManager); + return new TestOptimizationTracerManager.LockedManager(settings, agentWriter, scopeManager, statsd, runtimeMetrics, logSubmissionManager, telemetry, discoveryService, dataStreamsManager, gitMetadataTagsProvider, traceSampler, spanSampler, remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager, spanEventsManager, settingsManager); } - return new TestOptimizationTracerManager(settings, agentWriter, scopeManager, statsd, runtimeMetrics, logSubmissionManager, telemetry, discoveryService, dataStreamsManager, gitMetadataTagsProvider, traceSampler, spanSampler, remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager, spanEventsManager); + return new TestOptimizationTracerManager(settings, agentWriter, scopeManager, statsd, runtimeMetrics, logSubmissionManager, telemetry, discoveryService, dataStreamsManager, gitMetadataTagsProvider, traceSampler, spanSampler, remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager, spanEventsManager, settingsManager); } protected override ITelemetryController CreateTelemetryController(TracerSettings settings, IDiscoveryService discoveryService) diff --git a/tracer/src/Datadog.Trace/Tracer.cs b/tracer/src/Datadog.Trace/Tracer.cs index 96aa01feba98..3cdc7fcbbc19 100644 --- a/tracer/src/Datadog.Trace/Tracer.cs +++ b/tracer/src/Datadog.Trace/Tracer.cs @@ -52,7 +52,7 @@ public class Tracer : IDatadogTracer, IDatadogOpenTracingTracer /// The created will be scoped specifically to this instance. /// internal Tracer(TracerSettings settings, IAgentWriter agentWriter, ITraceSampler sampler, IScopeManager scopeManager, IDogStatsd statsd, ITelemetryController telemetry = null, IDiscoveryService discoveryService = null) - : this(TracerManagerFactory.Instance.CreateTracerManager(settings, agentWriter, sampler, scopeManager, statsd, runtimeMetrics: null, logSubmissionManager: null, telemetry: telemetry ?? NullTelemetryController.Instance, discoveryService ?? NullDiscoveryService.Instance, dataStreamsManager: null, remoteConfigurationManager: null, dynamicConfigurationManager: null, tracerFlareManager: null, spanEventsManager: null)) + : this(TracerManagerFactory.Instance.CreateTracerManager(settings, agentWriter, sampler, scopeManager, statsd, runtimeMetrics: null, logSubmissionManager: null, telemetry: telemetry ?? NullTelemetryController.Instance, discoveryService ?? NullDiscoveryService.Instance, dataStreamsManager: null, remoteConfigurationManager: null, dynamicConfigurationManager: null, tracerFlareManager: null, spanEventsManager: null, settingsManager: null)) { } diff --git a/tracer/src/Datadog.Trace/TracerManager.cs b/tracer/src/Datadog.Trace/TracerManager.cs index 89d28ad425ed..5e678fa164fd 100644 --- a/tracer/src/Datadog.Trace/TracerManager.cs +++ b/tracer/src/Datadog.Trace/TracerManager.cs @@ -71,6 +71,7 @@ public TracerManager( IDynamicConfigurationManager dynamicConfigurationManager, ITracerFlareManager tracerFlareManager, ISpanEventsManager spanEventsManager, + SettingsManager settingsManager, ITraceProcessor[] traceProcessors = null) { Settings = settings; @@ -99,6 +100,7 @@ public TracerManager( RemoteConfigurationManager = remoteConfigurationManager; DynamicConfigurationManager = dynamicConfigurationManager; TracerFlareManager = tracerFlareManager; + SettingsManager = settingsManager; SpanEventsManager = new SpanEventsManager(discoveryService); var schema = new NamingSchema(settings.MetadataSchemaVersion, settings.PeerServiceTagsEnabled, settings.RemoveClientServiceNamesEnabled, settings.MutableSettings.DefaultServiceName, settings.MutableSettings.ServiceNameMappings, settings.PeerServiceNameMappings); @@ -159,6 +161,8 @@ public static TracerManager Instance public ITracerFlareManager TracerFlareManager { get; } + public SettingsManager SettingsManager { get; } + public RuntimeMetricsWriter RuntimeMetrics { get; } public ISpanEventsManager SpanEventsManager { get; } diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index e967d6f7cf42..345b00ed9a66 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -69,7 +69,8 @@ internal TracerManager CreateTracerManager(TracerSettings settings, TracerManage remoteConfigurationManager: null, dynamicConfigurationManager: null, tracerFlareManager: null, - spanEventsManager: null); + spanEventsManager: null, + settingsManager: previous?.SettingsManager); try { @@ -108,7 +109,8 @@ internal TracerManager CreateTracerManager( IRemoteConfigurationManager remoteConfigurationManager, IDynamicConfigurationManager dynamicConfigurationManager, ITracerFlareManager tracerFlareManager, - ISpanEventsManager spanEventsManager) + ISpanEventsManager spanEventsManager, + SettingsManager settingsManager) { settings ??= TracerSettings.FromDefaultSourcesInternal(); var result = GlobalConfigurationSource.CreationResult; @@ -117,6 +119,9 @@ internal TracerManager CreateTracerManager( Log.Warning(result.Exception, "Failed to create the global configuration source with status: {Status} and error message: {ErrorMessage}", result.Result.ToString(), result.ErrorMessage); } + // TODO: Move the initial settings from TracerSettings to be created in SettingsManager here and not exposed in TracerSettings + settingsManager ??= new SettingsManager(settings, settings.InitialMutableSettings, settings.Exporter); + var libdatadogAvailaibility = LibDatadogAvailabilityHelper.IsLibDatadogAvailable; if (libdatadogAvailaibility.Exception is not null) { @@ -233,7 +238,8 @@ internal TracerManager CreateTracerManager( remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager, - spanEventsManager); + spanEventsManager, + settingsManager); } protected virtual ITelemetryController CreateTelemetryController(TracerSettings settings, IDiscoveryService discoveryService) @@ -268,8 +274,9 @@ protected virtual TracerManager CreateTracerManagerFrom( IRemoteConfigurationManager remoteConfigurationManager, IDynamicConfigurationManager dynamicConfigurationManager, ITracerFlareManager tracerFlareManager, - ISpanEventsManager spanEventsManager) - => new TracerManager(settings, agentWriter, scopeManager, statsd, runtimeMetrics, logSubmissionManager, telemetry, discoveryService, dataStreamsManager, gitMetadataTagsProvider, traceSampler, spanSampler, remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager, spanEventsManager); + ISpanEventsManager spanEventsManager, + SettingsManager settingsManager) + => new TracerManager(settings, agentWriter, scopeManager, statsd, runtimeMetrics, logSubmissionManager, telemetry, discoveryService, dataStreamsManager, gitMetadataTagsProvider, traceSampler, spanSampler, remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager, spanEventsManager, settingsManager); protected virtual ITraceSampler GetSampler(TracerSettings settings) { diff --git a/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs b/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs index a0f530a11e2c..5d8fd2a70c4c 100644 --- a/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs +++ b/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs @@ -142,7 +142,8 @@ private static TracerManager CreateTracerManager(TracerSettings settings) remoteConfigurationManager: null, dynamicConfigurationManager: null, tracerFlareManager: null, - spanEventsManager: null); + spanEventsManager: null, + settingsManager: null); static DirectLogSubmissionManager BuildLogSubmissionManager() => DirectLogSubmissionManager.Create( From 59cbb25da187654eb4307763f17242de6ae16927 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 22 Oct 2025 10:31:20 +0100 Subject: [PATCH 3/9] Move configure call out of remote config and config in code integrations --- .../Tracer/ConfigureIntegration.cs | 62 ++--------------- .../DynamicConfigurationManager.cs | 69 ++----------------- tracer/src/Datadog.Trace/TracerManager.cs | 23 ++++++- 3 files changed, 31 insertions(+), 123 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/ConfigureIntegration.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/ConfigureIntegration.cs index e1b546343979..5c8684949c0b 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/ConfigureIntegration.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/ConfigureIntegration.cs @@ -74,68 +74,14 @@ internal static void ConfigureSettingsWithManualOverrides(Dictionary Log.Error(t?.Exception, "Error updating dynamic configuration for debugger"), TaskContinuationOptions.OnlyOnFaulted); } diff --git a/tracer/src/Datadog.Trace/TracerManager.cs b/tracer/src/Datadog.Trace/TracerManager.cs index 5e678fa164fd..5a3688e6144f 100644 --- a/tracer/src/Datadog.Trace/TracerManager.cs +++ b/tracer/src/Datadog.Trace/TracerManager.cs @@ -670,7 +670,7 @@ private static TracerManager CreateInitializedTracer(TracerSettings settings, Tr if (_firstInitialization) { _firstInitialization = false; - OneTimeSetup(newManager.Settings); + OneTimeSetup(newManager.Settings, newManager.SettingsManager); } if (newManager.PerTraceSettings.Settings.StartupDiagnosticLogEnabled) @@ -681,7 +681,7 @@ private static TracerManager CreateInitializedTracer(TracerSettings settings, Tr return newManager; } - private static void OneTimeSetup(TracerSettings tracerSettings) + private static void OneTimeSetup(TracerSettings tracerSettings, SettingsManager settingsManager) { // Register callbacks to make sure we flush the traces before exiting LifetimeManager.Instance.AddAsyncShutdownTask(RunShutdownTasksAsync); @@ -691,6 +691,25 @@ private static void OneTimeSetup(TracerSettings tracerSettings) // Record the service discovery metadata ServiceDiscoveryHelper.StoreTracerMetadata(tracerSettings); + + // Register for rebuilding the settings on changes + // TODO: This is only temporary, we want to _stop_ rebuilding everything whenever settings change in the future + // We also don't bother to dispose this because we never unsubscribe + settingsManager.SubscribeToChanges(updatedSettings => + { + var newSettings = updatedSettings switch + { + { Exporter: { } e, Mutable: { } m } => Tracer.Instance.Settings with { Exporter = e, MutableSettings = m }, + { Exporter: { } e } => Tracer.Instance.Settings with { Exporter = e }, + { Mutable: { } m } => Tracer.Instance.Settings with { MutableSettings = m }, + _ => null, + }; + if (newSettings != null) + { + // Update the global instance + Trace.Tracer.Configure(newSettings); + } + }); } private static Task RunShutdownTasksAsync(Exception ex) => RunShutdownTasksAsync(_instance, _heartbeatTimer); From ef5f1e44fbaa544b7dce726d75f37d68e41dd5e4 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 22 Oct 2025 12:58:14 +0100 Subject: [PATCH 4/9] Move SettingsManager to live on TracerSettings for subseqent simplicity --- .../Ci/TestOptimizationTracerManager.cs | 10 +- .../TestOptimizationTracerManagerFactory.cs | 7 +- .../Tracer/ConfigureIntegration.cs | 2 +- .../DynamicConfigurationManager.cs | 2 +- .../Configuration/SettingsManager.cs | 325 +++++++++--------- .../Configuration/TracerSettings.cs | 5 +- tracer/src/Datadog.Trace/Tracer.cs | 2 +- tracer/src/Datadog.Trace/TracerManager.cs | 16 +- .../src/Datadog.Trace/TracerManagerFactory.cs | 17 +- .../TracerSettingsSettingManagerTests.cs | 26 ++ .../TracerManagerFactoryTests.cs | 3 +- 11 files changed, 221 insertions(+), 194 deletions(-) create mode 100644 tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsSettingManagerTests.cs diff --git a/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManager.cs b/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManager.cs index 6899c7579c72..f7f894912998 100644 --- a/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManager.cs +++ b/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManager.cs @@ -44,8 +44,7 @@ public TestOptimizationTracerManager( IRemoteConfigurationManager remoteConfigurationManager, IDynamicConfigurationManager dynamicConfigurationManager, ITracerFlareManager tracerFlareManager, - ISpanEventsManager spanEventsManager, - SettingsManager settingsManager) + ISpanEventsManager spanEventsManager) : base( settings, agentWriter, @@ -63,7 +62,6 @@ public TestOptimizationTracerManager( dynamicConfigurationManager, tracerFlareManager, spanEventsManager, - settingsManager, GetProcessors(settings.PartialFlushEnabled, agentWriter is CIVisibilityProtocolWriter)) { } @@ -162,8 +160,7 @@ public LockedManager( IRemoteConfigurationManager remoteConfigurationManager, IDynamicConfigurationManager dynamicConfigurationManager, ITracerFlareManager tracerFlareManager, - ISpanEventsManager spanEventsManager, - SettingsManager settingsManager) + ISpanEventsManager spanEventsManager) : base( settings, agentWriter, @@ -180,8 +177,7 @@ public LockedManager( remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager, - spanEventsManager, - settingsManager) + spanEventsManager) { } } diff --git a/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagerFactory.cs b/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagerFactory.cs index f477c326cffc..53798be38d6f 100644 --- a/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagerFactory.cs @@ -53,16 +53,15 @@ protected override TracerManager CreateTracerManagerFrom( IRemoteConfigurationManager remoteConfigurationManager, IDynamicConfigurationManager dynamicConfigurationManager, ITracerFlareManager tracerFlareManager, - ISpanEventsManager spanEventsManager, - SettingsManager settingsManager) + ISpanEventsManager spanEventsManager) { telemetry.RecordTestOptimizationSettings(_settings); if (_testOptimizationTracerManagement.UseLockedTracerManager) { - return new TestOptimizationTracerManager.LockedManager(settings, agentWriter, scopeManager, statsd, runtimeMetrics, logSubmissionManager, telemetry, discoveryService, dataStreamsManager, gitMetadataTagsProvider, traceSampler, spanSampler, remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager, spanEventsManager, settingsManager); + return new TestOptimizationTracerManager.LockedManager(settings, agentWriter, scopeManager, statsd, runtimeMetrics, logSubmissionManager, telemetry, discoveryService, dataStreamsManager, gitMetadataTagsProvider, traceSampler, spanSampler, remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager, spanEventsManager); } - return new TestOptimizationTracerManager(settings, agentWriter, scopeManager, statsd, runtimeMetrics, logSubmissionManager, telemetry, discoveryService, dataStreamsManager, gitMetadataTagsProvider, traceSampler, spanSampler, remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager, spanEventsManager, settingsManager); + return new TestOptimizationTracerManager(settings, agentWriter, scopeManager, statsd, runtimeMetrics, logSubmissionManager, telemetry, discoveryService, dataStreamsManager, gitMetadataTagsProvider, traceSampler, spanSampler, remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager, spanEventsManager); } protected override ITelemetryController CreateTelemetryController(TracerSettings settings, IDiscoveryService discoveryService) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/ConfigureIntegration.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/ConfigureIntegration.cs index 5c8684949c0b..13d9015ddb3a 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/ConfigureIntegration.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/ConfigureIntegration.cs @@ -78,7 +78,7 @@ internal static void ConfigureSettingsWithManualOverrides(Dictionary(); - private readonly TracerSettings _tracerSettings = tracerSettings; - private readonly List _subscribers = []; - private SettingChanges? _latest; - - /// - /// Gets the initial . On app startup, these will be the values read from - /// static sources. To subscribe to updates to these settings, from code or remote config, call . - /// - public MutableSettings InitialMutableSettings { get; } = initialMutable; - - /// - /// Gets the initial . On app startup, these will be the values read from - /// static sources. To subscribe to updates to these settings, from code or remote config, call . - /// - public ExporterSettings InitialExporterSettings { get; } = initialExporter; - - /// - /// Subscribe to changes in and/or . - /// Called whenever these settings change. If the settings have already changed when - /// is called, is invoked immediately with the latest configuration. - /// Also note that calling twice with the same callback - /// will invoke the callback twice. - /// - /// The method to invoke - /// An that should be disposed to unsubscribe - public IDisposable SubscribeToChanges(Action callback) + internal class SettingsManager( + TracerSettings tracerSettings, + MutableSettings initialMutable, + ExporterSettings initialExporter) { - var subscription = new SettingChangeSubscription(this, callback); - lock (_subscribers) - { - _subscribers.Add(subscription); - } + private readonly TracerSettings _tracerSettings = tracerSettings; + private readonly List _subscribers = []; + private SettingChanges? _latest; + + /// + /// Gets the initial . On app startup, these will be the values read from + /// static sources. To subscribe to updates to these settings, from code or remote config, call . + /// + public MutableSettings InitialMutableSettings { get; } = initialMutable; + + /// + /// Gets the initial . On app startup, these will be the values read from + /// static sources. To subscribe to updates to these settings, from code or remote config, call . + /// + public ExporterSettings InitialExporterSettings { get; } = initialExporter; - if (Volatile.Read(ref _latest) is { } currentConfig) + /// + /// Subscribe to changes in and/or . + /// Called whenever these settings change. If the settings have already changed when + /// is called, is invoked immediately with the latest configuration. + /// Also note that calling twice with the same callback + /// will invoke the callback twice. + /// + /// The method to invoke + /// An that should be disposed to unsubscribe + public IDisposable SubscribeToChanges(Action callback) { - try + var subscription = new SettingChangeSubscription(this, callback); + lock (_subscribers) { - // If we already have updates, call this immediately - callback(currentConfig); + _subscribers.Add(subscription); } - catch (Exception ex) + + if (Volatile.Read(ref _latest) is { } currentConfig) { - Log.Error(ex, "Error notifying subscriber of updated MutableSettings during subscribe"); + try + { + // If we already have updates, call this immediately + callback(currentConfig); + } + catch (Exception ex) + { + Log.Error(ex, "Error notifying subscriber of updated MutableSettings during subscribe"); + } } - } - return subscription; - } - - /// - /// Regenerate the application's new based on runtime configuration sources - /// - /// An for dynamic config via remote config - /// An for manual configuration (in code) - /// The central to report config telemetry updates to - /// True if changes were detected and consumers were updated, false otherwise - public bool UpdateSettings( - IConfigurationSource dynamicConfigSource, - ManualInstrumentationConfigurationSourceBase manualSource, - IConfigurationTelemetry centralTelemetry) - { - if (BuildNewSettings(dynamicConfigSource, manualSource, centralTelemetry) is { } newSettings) - { - NotifySubscribers(newSettings); - return true; + return subscription; } - return false; - } - - // Internal for testing - internal SettingChanges? BuildNewSettings( - IConfigurationSource dynamicConfigSource, - ManualInstrumentationConfigurationSourceBase manualSource, - IConfigurationTelemetry centralTelemetry) - { - var initialSettings = manualSource.UseDefaultSources - ? InitialMutableSettings - : MutableSettings.CreateWithoutDefaultSources(_tracerSettings); - - var current = Volatile.Read(ref _latest); - var currentMutable = current?.Mutable ?? InitialMutableSettings; - var currentExporter = current?.Exporter ?? InitialExporterSettings; - - var telemetry = new ConfigurationTelemetry(); - var newMutableSettings = MutableSettings.CreateUpdatedMutableSettings( - dynamicConfigSource, - manualSource, - initialSettings, - _tracerSettings, - telemetry, - new OverrideErrorLog()); // TODO: We'll later report these - - // The only exporter setting we currently _allow_ to change is the AgentUri, but if that does change, - // it can mean that _everything_ about the exporter settings changes. To minimize the work to do, and - // to simplify comparisons, we try to read the agent url from the manual setting. If it's missing, not - // set, or unchanged, there's no need to update the exporter settings. - // We only technically need to do this today if _manual_ config changes, not if remote config changes, - // but for simplicity we don't distinguish currently. - var exporterTelemetry = new ConfigurationTelemetry(); - var newRawExporterSettings = ExporterSettings.Raw.CreateUpdatedFromManualConfig( - currentExporter.RawSettings, - manualSource, - exporterTelemetry, - manualSource.UseDefaultSources); - - var isSameMutableSettings = currentMutable.Equals(newMutableSettings); - var isSameExporterSettings = currentExporter.RawSettings.Equals(newRawExporterSettings); - - if (isSameMutableSettings && isSameExporterSettings) + /// + /// Regenerate the application's new and + /// based on runtime configuration sources. + /// + /// An for dynamic config via remote config + /// An for manual configuration (in code) + /// The central to report config telemetry updates to + /// True if changes were detected and consumers were updated, false otherwise + public bool UpdateSettings( + IConfigurationSource dynamicConfigSource, + ManualInstrumentationConfigurationSourceBase manualSource, + IConfigurationTelemetry centralTelemetry) { - Log.Debug("No changes detected in the new configuration"); - // Even though there were no "real" changes, there may be _effective_ changes in telemetry that - // need to be recorded (e.g. the customer set the value in code, but it was already set via - // env vars). We _should_ record exporter settings too, but that introduces a bunch of complexity - // which we'll resolve later anyway, so just have that gap for now (it's very niche). - // If there are changes, they're recorded automatically in ConfigureInternal - telemetry.CopyTo(centralTelemetry); - return null; + if (BuildNewSettings(dynamicConfigSource, manualSource, centralTelemetry) is { } newSettings) + { + NotifySubscribers(newSettings); + return true; + } + + return false; } - Log.Information("Notifying consumers of new settings"); - var updatedMutableSettings = isSameMutableSettings ? null : newMutableSettings; - var updatedExporterSettings = isSameExporterSettings ? null : new ExporterSettings(newRawExporterSettings, exporterTelemetry); + // Internal for testing + internal SettingChanges? BuildNewSettings( + IConfigurationSource dynamicConfigSource, + ManualInstrumentationConfigurationSourceBase manualSource, + IConfigurationTelemetry centralTelemetry) + { + var initialSettings = manualSource.UseDefaultSources + ? InitialMutableSettings + : MutableSettings.CreateWithoutDefaultSources(_tracerSettings); + + var current = _latest; + var currentMutable = current?.UpdatedMutable ?? current?.PreviousMutable ?? InitialMutableSettings; + var currentExporter = current?.UpdatedExporter ?? current?.PreviousExporter ?? InitialExporterSettings; + + var telemetry = new ConfigurationTelemetry(); + var newMutableSettings = MutableSettings.CreateUpdatedMutableSettings( + dynamicConfigSource, + manualSource, + initialSettings, + _tracerSettings, + telemetry, + new OverrideErrorLog()); // TODO: We'll later report these + + // The only exporter setting we currently _allow_ to change is the AgentUri, but if that does change, + // it can mean that _everything_ about the exporter settings changes. To minimize the work to do, and + // to simplify comparisons, we try to read the agent url from the manual setting. If it's missing, not + // set, or unchanged, there's no need to update the exporter settings. + // We only technically need to do this today if _manual_ config changes, not if remote config changes, + // but for simplicity we don't distinguish currently. + var exporterTelemetry = new ConfigurationTelemetry(); + var newRawExporterSettings = ExporterSettings.Raw.CreateUpdatedFromManualConfig( + currentExporter.RawSettings, + manualSource, + exporterTelemetry, + manualSource.UseDefaultSources); + + var isSameMutableSettings = currentMutable.Equals(newMutableSettings); + var isSameExporterSettings = currentExporter.RawSettings.Equals(newRawExporterSettings); + + if (isSameMutableSettings && isSameExporterSettings) + { + Log.Debug("No changes detected in the new configuration"); + // Even though there were no "real" changes, there may be _effective_ changes in telemetry that + // need to be recorded (e.g. the customer set the value in code, but it was already set via + // env vars). We _should_ record exporter settings too, but that introduces a bunch of complexity + // which we'll resolve later anyway, so just have that gap for now (it's very niche). + // If there are changes, they're recorded automatically in ConfigureInternal + telemetry.CopyTo(centralTelemetry); + return null; + } - return new SettingChanges(updatedMutableSettings, updatedExporterSettings); - } + Log.Information("Notifying consumers of new settings"); + var updatedMutableSettings = isSameMutableSettings ? null : newMutableSettings; + var updatedExporterSettings = isSameExporterSettings ? null : new ExporterSettings(newRawExporterSettings, exporterTelemetry); - private void NotifySubscribers(SettingChanges settings) - { - // Strictly, for safety, we only need to lock in the subscribers list access. However, - // there's nothing to prevent NotifySubscribers being called concurrently, - // which could result in weird out-of-order notifications for customers. So for simplicity - // we just lock the whole method to ensure serialized updates. + return new SettingChanges(updatedMutableSettings, updatedExporterSettings, currentMutable, currentExporter); + } - lock (_subscribers) + private void NotifySubscribers(SettingChanges settings) { - var subscribers = _subscribers; - Volatile.Write(ref _latest, settings); + // Strictly, for safety, we only need to lock in the subscribers list access. However, + // there's nothing to prevent NotifySubscribers being called concurrently, + // which could result in weird out-of-order notifications for customers. So for simplicity + // we just lock the whole method to ensure serialized updates. - foreach (var subscriber in subscribers) + lock (_subscribers) { - try - { - subscriber.Notify(settings); - } - catch (Exception ex) + var subscribers = _subscribers; + Volatile.Write(ref _latest, settings); + + foreach (var subscriber in subscribers) { - Log.Error(ex, "Error notifying subscriber of MutableSettings change"); + try + { + subscriber.Notify(settings); + } + catch (Exception ex) + { + Log.Error(ex, "Error notifying subscriber of MutableSettings change"); + } } } } - } - private sealed class SettingChangeSubscription(SettingsManager owner, Action notify) : IDisposable - { - private readonly SettingsManager _owner = owner; + private sealed class SettingChangeSubscription(SettingsManager owner, Action notify) : IDisposable + { + private readonly SettingsManager _owner = owner; - public Action Notify { get; } = notify; + public Action Notify { get; } = notify; - public void Dispose() - { - lock (_owner._subscribers) + public void Dispose() { - _owner._subscribers.Remove(this); + lock (_owner._subscribers) + { + _owner._subscribers.Remove(this); + } } } - } - - public sealed class SettingChanges(MutableSettings? mutableSettings, ExporterSettings? exporterSettings) - { - /// - /// Gets the new , if they have changed. - /// If there are no changes, returns null. - /// - public MutableSettings? Mutable { get; } = mutableSettings; - /// - /// Gets the new , if they have changed. - /// If there are no changes, returns null. - /// - public ExporterSettings? Exporter { get; } = exporterSettings; + public sealed class SettingChanges(MutableSettings? updatedMutable, ExporterSettings? updatedExporter, MutableSettings previousMutable, ExporterSettings previousExporter) + { + /// + /// Gets the new , if they have changed. + /// If there are no changes, returns null. + /// + public MutableSettings? UpdatedMutable { get; } = updatedMutable; + + /// + /// Gets the new , if they have changed. + /// If there are no changes, returns null. + /// + public ExporterSettings? UpdatedExporter { get; } = updatedExporter; + + /// + /// Gets the previous , prior to this update. + /// + public MutableSettings PreviousMutable { get; } = previousMutable; + + /// + /// Gets the previous , prior to this update. + /// + public ExporterSettings PreviousExporter { get; } = previousExporter; + } } } diff --git a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs index 79eed12b3322..b64595f40f34 100644 --- a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs +++ b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs @@ -30,7 +30,7 @@ namespace Datadog.Trace.Configuration /// /// Contains Tracer settings. /// - public record TracerSettings + public partial record TracerSettings { private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); private static readonly HashSet DefaultExperimentalFeatures = ["DD_TAGS", ConfigurationKeys.PropagateProcessTags]; @@ -729,6 +729,7 @@ not null when string.Equals(value, "otlp", StringComparison.OrdinalIgnoreCase) = InitialMutableSettings = MutableSettings.CreateInitialMutableSettings(source, telemetry, errorLog, this); MutableSettings = InitialMutableSettings; + Manager = new(this, InitialMutableSettings, Exporter); } internal bool IsRunningInCiVisibility { get; } @@ -1320,6 +1321,8 @@ not null when string.Equals(value, "otlp", StringComparison.OrdinalIgnoreCase) = /// public int PartialFlushMinSpans { get; } + internal SettingsManager Manager { get; } + internal List JsonConfigurationFilePaths { get; } = new(); /// diff --git a/tracer/src/Datadog.Trace/Tracer.cs b/tracer/src/Datadog.Trace/Tracer.cs index 3cdc7fcbbc19..96aa01feba98 100644 --- a/tracer/src/Datadog.Trace/Tracer.cs +++ b/tracer/src/Datadog.Trace/Tracer.cs @@ -52,7 +52,7 @@ public class Tracer : IDatadogTracer, IDatadogOpenTracingTracer /// The created will be scoped specifically to this instance. /// internal Tracer(TracerSettings settings, IAgentWriter agentWriter, ITraceSampler sampler, IScopeManager scopeManager, IDogStatsd statsd, ITelemetryController telemetry = null, IDiscoveryService discoveryService = null) - : this(TracerManagerFactory.Instance.CreateTracerManager(settings, agentWriter, sampler, scopeManager, statsd, runtimeMetrics: null, logSubmissionManager: null, telemetry: telemetry ?? NullTelemetryController.Instance, discoveryService ?? NullDiscoveryService.Instance, dataStreamsManager: null, remoteConfigurationManager: null, dynamicConfigurationManager: null, tracerFlareManager: null, spanEventsManager: null, settingsManager: null)) + : this(TracerManagerFactory.Instance.CreateTracerManager(settings, agentWriter, sampler, scopeManager, statsd, runtimeMetrics: null, logSubmissionManager: null, telemetry: telemetry ?? NullTelemetryController.Instance, discoveryService ?? NullDiscoveryService.Instance, dataStreamsManager: null, remoteConfigurationManager: null, dynamicConfigurationManager: null, tracerFlareManager: null, spanEventsManager: null)) { } diff --git a/tracer/src/Datadog.Trace/TracerManager.cs b/tracer/src/Datadog.Trace/TracerManager.cs index 5a3688e6144f..51ba3f9caacf 100644 --- a/tracer/src/Datadog.Trace/TracerManager.cs +++ b/tracer/src/Datadog.Trace/TracerManager.cs @@ -71,7 +71,6 @@ public TracerManager( IDynamicConfigurationManager dynamicConfigurationManager, ITracerFlareManager tracerFlareManager, ISpanEventsManager spanEventsManager, - SettingsManager settingsManager, ITraceProcessor[] traceProcessors = null) { Settings = settings; @@ -100,7 +99,6 @@ public TracerManager( RemoteConfigurationManager = remoteConfigurationManager; DynamicConfigurationManager = dynamicConfigurationManager; TracerFlareManager = tracerFlareManager; - SettingsManager = settingsManager; SpanEventsManager = new SpanEventsManager(discoveryService); var schema = new NamingSchema(settings.MetadataSchemaVersion, settings.PeerServiceTagsEnabled, settings.RemoveClientServiceNamesEnabled, settings.MutableSettings.DefaultServiceName, settings.MutableSettings.ServiceNameMappings, settings.PeerServiceNameMappings); @@ -161,8 +159,6 @@ public static TracerManager Instance public ITracerFlareManager TracerFlareManager { get; } - public SettingsManager SettingsManager { get; } - public RuntimeMetricsWriter RuntimeMetrics { get; } public ISpanEventsManager SpanEventsManager { get; } @@ -670,7 +666,7 @@ private static TracerManager CreateInitializedTracer(TracerSettings settings, Tr if (_firstInitialization) { _firstInitialization = false; - OneTimeSetup(newManager.Settings, newManager.SettingsManager); + OneTimeSetup(newManager.Settings); } if (newManager.PerTraceSettings.Settings.StartupDiagnosticLogEnabled) @@ -681,7 +677,7 @@ private static TracerManager CreateInitializedTracer(TracerSettings settings, Tr return newManager; } - private static void OneTimeSetup(TracerSettings tracerSettings, SettingsManager settingsManager) + private static void OneTimeSetup(TracerSettings tracerSettings) { // Register callbacks to make sure we flush the traces before exiting LifetimeManager.Instance.AddAsyncShutdownTask(RunShutdownTasksAsync); @@ -695,13 +691,13 @@ private static void OneTimeSetup(TracerSettings tracerSettings, SettingsManager // Register for rebuilding the settings on changes // TODO: This is only temporary, we want to _stop_ rebuilding everything whenever settings change in the future // We also don't bother to dispose this because we never unsubscribe - settingsManager.SubscribeToChanges(updatedSettings => + tracerSettings.Manager.SubscribeToChanges(updatedSettings => { var newSettings = updatedSettings switch { - { Exporter: { } e, Mutable: { } m } => Tracer.Instance.Settings with { Exporter = e, MutableSettings = m }, - { Exporter: { } e } => Tracer.Instance.Settings with { Exporter = e }, - { Mutable: { } m } => Tracer.Instance.Settings with { MutableSettings = m }, + { UpdatedExporter: { } e, UpdatedMutable: { } m } => Tracer.Instance.Settings with { Exporter = e, MutableSettings = m }, + { UpdatedExporter: { } e } => Tracer.Instance.Settings with { Exporter = e }, + { UpdatedMutable: { } m } => Tracer.Instance.Settings with { MutableSettings = m }, _ => null, }; if (newSettings != null) diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index 345b00ed9a66..e967d6f7cf42 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -69,8 +69,7 @@ internal TracerManager CreateTracerManager(TracerSettings settings, TracerManage remoteConfigurationManager: null, dynamicConfigurationManager: null, tracerFlareManager: null, - spanEventsManager: null, - settingsManager: previous?.SettingsManager); + spanEventsManager: null); try { @@ -109,8 +108,7 @@ internal TracerManager CreateTracerManager( IRemoteConfigurationManager remoteConfigurationManager, IDynamicConfigurationManager dynamicConfigurationManager, ITracerFlareManager tracerFlareManager, - ISpanEventsManager spanEventsManager, - SettingsManager settingsManager) + ISpanEventsManager spanEventsManager) { settings ??= TracerSettings.FromDefaultSourcesInternal(); var result = GlobalConfigurationSource.CreationResult; @@ -119,9 +117,6 @@ internal TracerManager CreateTracerManager( Log.Warning(result.Exception, "Failed to create the global configuration source with status: {Status} and error message: {ErrorMessage}", result.Result.ToString(), result.ErrorMessage); } - // TODO: Move the initial settings from TracerSettings to be created in SettingsManager here and not exposed in TracerSettings - settingsManager ??= new SettingsManager(settings, settings.InitialMutableSettings, settings.Exporter); - var libdatadogAvailaibility = LibDatadogAvailabilityHelper.IsLibDatadogAvailable; if (libdatadogAvailaibility.Exception is not null) { @@ -238,8 +233,7 @@ internal TracerManager CreateTracerManager( remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager, - spanEventsManager, - settingsManager); + spanEventsManager); } protected virtual ITelemetryController CreateTelemetryController(TracerSettings settings, IDiscoveryService discoveryService) @@ -274,9 +268,8 @@ protected virtual TracerManager CreateTracerManagerFrom( IRemoteConfigurationManager remoteConfigurationManager, IDynamicConfigurationManager dynamicConfigurationManager, ITracerFlareManager tracerFlareManager, - ISpanEventsManager spanEventsManager, - SettingsManager settingsManager) - => new TracerManager(settings, agentWriter, scopeManager, statsd, runtimeMetrics, logSubmissionManager, telemetry, discoveryService, dataStreamsManager, gitMetadataTagsProvider, traceSampler, spanSampler, remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager, spanEventsManager, settingsManager); + ISpanEventsManager spanEventsManager) + => new TracerManager(settings, agentWriter, scopeManager, statsd, runtimeMetrics, logSubmissionManager, telemetry, discoveryService, dataStreamsManager, gitMetadataTagsProvider, traceSampler, spanSampler, remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager, spanEventsManager); protected virtual ITraceSampler GetSampler(TracerSettings settings) { diff --git a/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsSettingManagerTests.cs b/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsSettingManagerTests.cs new file mode 100644 index 000000000000..a4fc267033fd --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsSettingManagerTests.cs @@ -0,0 +1,26 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using Datadog.Trace.Configuration; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.Configuration; + +public class TracerSettingsSettingManagerTests +{ + [Fact] + public void UpdatingTracerSettingsDoesNotReplaceSettingsManager() + { + var tracerSettings = TracerSettings.Create([]); + tracerSettings.MutableSettings.Should().BeSameAs(tracerSettings.InitialMutableSettings); + + var originalManager = tracerSettings.Manager; + var newSettings = tracerSettings with { MutableSettings = MutableSettings.CreateForTesting(tracerSettings, []) }; + + newSettings.MutableSettings.Should().NotBeSameAs(newSettings.InitialMutableSettings); + newSettings.Manager.Should().BeSameAs(originalManager); + } +} diff --git a/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs b/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs index 5d8fd2a70c4c..a0f530a11e2c 100644 --- a/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs +++ b/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs @@ -142,8 +142,7 @@ private static TracerManager CreateTracerManager(TracerSettings settings) remoteConfigurationManager: null, dynamicConfigurationManager: null, tracerFlareManager: null, - spanEventsManager: null, - settingsManager: null); + spanEventsManager: null); static DirectLogSubmissionManager BuildLogSubmissionManager() => DirectLogSubmissionManager.Create( From c969f420ae607964d13484b15c4acf9101767f69 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 22 Oct 2025 11:19:51 +0100 Subject: [PATCH 5/9] Add helper for creating mutablesettings in tests --- tracer/src/Datadog.Trace/Configuration/MutableSettings.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tracer/src/Datadog.Trace/Configuration/MutableSettings.cs b/tracer/src/Datadog.Trace/Configuration/MutableSettings.cs index df7516c7d0f6..7881d83c2199 100644 --- a/tracer/src/Datadog.Trace/Configuration/MutableSettings.cs +++ b/tracer/src/Datadog.Trace/Configuration/MutableSettings.cs @@ -1087,6 +1087,13 @@ public static MutableSettings CreateWithoutDefaultSources(TracerSettings tracerS new OverrideErrorLog(), tracerSettings); + public static MutableSettings CreateForTesting(TracerSettings tracerSettings, Dictionary settings) + => CreateInitialMutableSettings( + new DictionaryConfigurationSource(settings.ToDictionary(x => x.Key, x => x.Value?.ToString()!)), + new ConfigurationTelemetry(), + new OverrideErrorLog(), + tracerSettings); + private static ConfigurationBuilder.ClassConfigurationResultWithKey> RemapOtelTags( in ConfigurationBuilder.ClassConfigurationResultWithKey> original) { From 5d98cbd18dbdf74c73e060b8415ce45515234dd5 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 22 Oct 2025 14:47:48 +0100 Subject: [PATCH 6/9] Refactor DynamicConfigurationManager to make it testable --- .../DynamicConfigurationManager.cs | 13 +-- .../DynamicConfigurationTests.cs | 83 ++++++++++++------- 2 files changed, 61 insertions(+), 35 deletions(-) diff --git a/tracer/src/Datadog.Trace/Configuration/DynamicConfigurationManager.cs b/tracer/src/Datadog.Trace/Configuration/DynamicConfigurationManager.cs index cf77f7a28424..6dcde2abe5c9 100644 --- a/tracer/src/Datadog.Trace/Configuration/DynamicConfigurationManager.cs +++ b/tracer/src/Datadog.Trace/Configuration/DynamicConfigurationManager.cs @@ -67,12 +67,12 @@ public void Dispose() } } - internal static void OnlyForTests_ApplyConfiguration(IConfigurationSource dynamicConfig) + internal static void OnlyForTests_ApplyConfiguration(IConfigurationSource dynamicConfig, TracerSettings tracerSettings) { - OnConfigurationChanged(dynamicConfig); + OnConfigurationChanged(dynamicConfig, tracerSettings); } - private static void OnConfigurationChanged(IConfigurationSource dynamicConfig) + private static void OnConfigurationChanged(IConfigurationSource dynamicConfig, TracerSettings tracerSettings) { var manualConfig = GlobalConfigurationSource.ManualConfigurationSource; @@ -80,7 +80,7 @@ private static void OnConfigurationChanged(IConfigurationSource dynamicConfig) // so that it can be picked up by other configuration updaters, e.g. config in code GlobalConfigurationSource.UpdateDynamicConfigConfigurationSource(dynamicConfig); - var wasUpdated = Tracer.Instance.Settings.Manager.UpdateSettings(dynamicConfig, manualConfig, TelemetryFactory.Config); + var wasUpdated = tracerSettings.Manager.UpdateSettings(dynamicConfig, manualConfig, TelemetryFactory.Config); if (wasUpdated) { Log.Information("Setting updates made via configuration in code were applied"); @@ -190,7 +190,8 @@ private ApplyDetails[] ConfigurationUpdated( private void ApplyMergedConfiguration(List remoteConfigurations) { // Get current service/environment for filtering - var currentSettings = Tracer.Instance.CurrentTraceSettings.Settings; + var tracer = Tracer.Instance; + var currentSettings = tracer.CurrentTraceSettings.Settings; var mergedConfigJToken = ApmTracingConfigMerger.MergeConfigurations( remoteConfigurations, @@ -199,7 +200,7 @@ private void ApplyMergedConfiguration(List remoteConfigurat var configurationSource = new DynamicConfigConfigurationSource(mergedConfigJToken, ConfigurationOrigins.RemoteConfig); - OnConfigurationChanged(configurationSource); + OnConfigurationChanged(configurationSource, tracer.Settings); } } } diff --git a/tracer/test/Datadog.Trace.Tests/Configuration/DynamicConfigurationTests.cs b/tracer/test/Datadog.Trace.Tests/Configuration/DynamicConfigurationTests.cs index 97e1f3afb1c1..dfb699fd684a 100644 --- a/tracer/test/Datadog.Trace.Tests/Configuration/DynamicConfigurationTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Configuration/DynamicConfigurationTests.cs @@ -29,7 +29,7 @@ public void ApplyServiceMappingToNewTraces() Tracer.Instance.CurrentTraceSettings.GetServiceName("test") .Should().Be($"{Tracer.Instance.DefaultServiceName}-test"); - DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_service_mapping", "test:ok"))); + DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_service_mapping", "test:ok")), Tracer.Instance.Settings); Tracer.Instance.CurrentTraceSettings.GetServiceName("test") .Should().Be($"{Tracer.Instance.DefaultServiceName}-test", "the old configuration should be used inside of the active trace"); @@ -45,13 +45,13 @@ public void ApplyConfigurationTwice() { var tracer = TracerManager.Instance; - DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_sampling_rate", 0.4))); + DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_sampling_rate", 0.4)), tracer.Settings); var newTracer = TracerManager.Instance; newTracer.Should().NotBeSameAs(tracer); - DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_sampling_rate", 0.4))); + DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_sampling_rate", 0.4)), tracer.Settings); TracerManager.Instance.Should().BeSameAs(newTracer); } @@ -60,14 +60,17 @@ public void ApplyConfigurationTwice() public void ApplyTagsToDirectLogs() { var tracerSettings = TracerSettings.Create(new() { { ConfigurationKeys.GlobalTags, "key1:value1" } }); - TracerManager.ReplaceGlobalManager(tracerSettings, TracerManagerFactory.Instance); - TracerManager.Instance.DirectLogSubmission.Formatter.Tags.Should().Be("key1:value1"); + // emulate the one-time subscribe that TracerManager.Instance does + var tracerManager = TracerManagerFactory.Instance.CreateTracerManager(tracerSettings, null); + using var sub = tracerSettings.Manager.SubscribeToChanges(x => tracerManager = UpdateTracerManager(x, tracerSettings, tracerManager)); + + tracerManager.DirectLogSubmission.Formatter.Tags.Should().Be("key1:value1"); var configBuilder = CreateConfig(("tracing_tags", new[] { "key2:value2" })); - DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(configBuilder); + DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(configBuilder, tracerSettings); - TracerManager.Instance.DirectLogSubmission.Formatter.Tags.Should().Be("key2:value2"); + tracerManager.DirectLogSubmission.Formatter.Tags.Should().Be("key2:value2"); } [Fact] @@ -79,14 +82,15 @@ public void DoesNotOverrideDirectLogsTags() { ConfigurationKeys.DirectLogSubmission.EnabledIntegrations, "xunit" }, { ConfigurationKeys.GlobalTags, "key2:value2" }, }); - TracerManager.ReplaceGlobalManager(tracerSettings, TracerManagerFactory.Instance); + var tracerManager = TracerManagerFactory.Instance.CreateTracerManager(tracerSettings, null); + using var sub = tracerSettings.Manager.SubscribeToChanges(x => tracerManager = UpdateTracerManager(x, tracerSettings, tracerManager)); - TracerManager.Instance.DirectLogSubmission.Formatter.Tags.Should().Be("key1:value1"); + tracerManager.DirectLogSubmission.Formatter.Tags.Should().Be("key1:value1"); var configBuilder = CreateConfig(("tracing_tags", new[] { "key3:value3" })); - DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(configBuilder); + DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(configBuilder, tracerSettings); - TracerManager.Instance.DirectLogSubmission.Formatter.Tags.Should().Be("key1:value1"); + tracerManager.DirectLogSubmission.Formatter.Tags.Should().Be("key1:value1"); } [Fact] @@ -97,32 +101,34 @@ public void DoesNotReplaceRuntimeMetricsWriter() { ConfigurationKeys.RuntimeMetricsEnabled, "true" }, { ConfigurationKeys.GlobalTags, "key1:value1" }, }); - TracerManager.ReplaceGlobalManager(tracerSettings, TracerManagerFactory.Instance); + var tracerManager = TracerManagerFactory.Instance.CreateTracerManager(tracerSettings, null); + using var sub = tracerSettings.Manager.SubscribeToChanges(x => tracerManager = UpdateTracerManager(x, tracerSettings, tracerManager)); - var previousRuntimeMetrics = TracerManager.Instance.RuntimeMetrics; + var previousRuntimeMetrics = tracerManager.RuntimeMetrics; var configBuilder = CreateConfig(("tracing_tags", new[] { "key2:value2" })); - DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(configBuilder); + DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(configBuilder, tracerSettings); - TracerManager.Instance.RuntimeMetrics.Should().Be(previousRuntimeMetrics); + tracerManager.RuntimeMetrics.Should().Be(previousRuntimeMetrics); } [Fact] public void EnableTracing() { var tracerSettings = new TracerSettings(); - TracerManager.ReplaceGlobalManager(tracerSettings, TracerManagerFactory.Instance); + var tracerManager = TracerManagerFactory.Instance.CreateTracerManager(tracerSettings, null); + using var sub = tracerSettings.Manager.SubscribeToChanges(x => tracerManager = UpdateTracerManager(x, tracerSettings, tracerManager)); // tracing is enabled by default - TracerManager.Instance.Settings.TraceEnabled.Should().BeTrue(); + tracerManager.Settings.TraceEnabled.Should().BeTrue(); // disable "remotely" - DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_enabled", false))); - TracerManager.Instance.Settings.TraceEnabled.Should().BeFalse(); + DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_enabled", false)), tracerSettings); + tracerManager.Settings.TraceEnabled.Should().BeFalse(); // re-enable "remotely" - DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_enabled", true))); - TracerManager.Instance.Settings.TraceEnabled.Should().BeTrue(); + DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_enabled", true)), tracerSettings); + tracerManager.Settings.TraceEnabled.Should().BeTrue(); } [Fact] @@ -141,12 +147,13 @@ public void SetSamplingRules() { "DD_TRACE_SAMPLING_RULES", localSamplingRulesJson } }); - TracerManager.ReplaceGlobalManager(tracerSettings, TracerManagerFactory.Instance); + var tracerManager = TracerManagerFactory.Instance.CreateTracerManager(tracerSettings, null); + using var sub = tracerSettings.Manager.SubscribeToChanges(x => tracerManager = UpdateTracerManager(x, tracerSettings, tracerManager)); - TracerManager.Instance.Settings.CustomSamplingRules.Should().Be(localSamplingRulesJson); - TracerManager.Instance.Settings.CustomSamplingRulesIsRemote.Should().BeFalse(); + tracerManager.Settings.CustomSamplingRules.Should().Be(localSamplingRulesJson); + tracerManager.Settings.CustomSamplingRulesIsRemote.Should().BeFalse(); - var rules = ((TraceSampler)TracerManager.Instance.PerTraceSettings.TraceSampler)!.GetRules(); + var rules = ((TraceSampler)tracerManager.PerTraceSettings.TraceSampler)!.GetRules(); rules.Should() .BeEquivalentTo( @@ -171,13 +178,13 @@ public void SetSamplingRules() }; var configBuilder = CreateConfig(("tracing_sampling_rules", remoteSamplingRulesConfig)); - DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(configBuilder); + DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(configBuilder, tracerSettings); var remoteSamplingRulesJson = JsonConvert.SerializeObject(remoteSamplingRulesConfig); - TracerManager.Instance.Settings.CustomSamplingRules.Should().Be(remoteSamplingRulesJson); - TracerManager.Instance.Settings.CustomSamplingRulesIsRemote.Should().BeTrue(); + tracerManager.Settings.CustomSamplingRules.Should().Be(remoteSamplingRulesJson); + tracerManager.Settings.CustomSamplingRulesIsRemote.Should().BeTrue(); - rules = ((TraceSampler)TracerManager.Instance.PerTraceSettings.TraceSampler)!.GetRules(); + rules = ((TraceSampler)tracerManager.PerTraceSettings.TraceSampler)!.GetRules(); // new list should include the remote rules, not the local rules rules.Should() @@ -271,5 +278,23 @@ private static DynamicConfigConfigurationSource CreateConfig(params (string Key, return new DynamicConfigConfigurationSource(configObj, ConfigurationOrigins.RemoteConfig); } + + private static TracerManager UpdateTracerManager(TracerSettings.SettingsManager.SettingChanges updates, TracerSettings settings, TracerManager tracerManager) + { + var newSettings = updates switch + { + { UpdatedExporter: { } e, UpdatedMutable: { } m } => settings with { Exporter = e, MutableSettings = m }, + { UpdatedExporter: { } e } => settings with { Exporter = e }, + { UpdatedMutable: { } m } => settings with { MutableSettings = m }, + _ => null, + }; + + if (newSettings != null) + { + tracerManager = TracerManagerFactory.Instance.CreateTracerManager(newSettings, tracerManager); + } + + return tracerManager; + } } } From 1b480ca728356392bd74b04aa09ab20cb09c54fe Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Tue, 28 Oct 2025 17:23:14 +0000 Subject: [PATCH 7/9] PR feedback - Ensure subscribing doesn't receive out-of-order updates - Fix log message --- .../DynamicConfigurationManager.cs | 2 +- .../Configuration/SettingsManager.cs | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tracer/src/Datadog.Trace/Configuration/DynamicConfigurationManager.cs b/tracer/src/Datadog.Trace/Configuration/DynamicConfigurationManager.cs index 6dcde2abe5c9..f046a8293ed0 100644 --- a/tracer/src/Datadog.Trace/Configuration/DynamicConfigurationManager.cs +++ b/tracer/src/Datadog.Trace/Configuration/DynamicConfigurationManager.cs @@ -83,7 +83,7 @@ private static void OnConfigurationChanged(IConfigurationSource dynamicConfig, T var wasUpdated = tracerSettings.Manager.UpdateSettings(dynamicConfig, manualConfig, TelemetryFactory.Config); if (wasUpdated) { - Log.Information("Setting updates made via configuration in code were applied"); + Log.Information("Setting updates made via dynamic configuration were applied"); } // TODO: This might not record the config in the correct order in future, but would require diff --git a/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs b/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs index ede3e65dc3d7..7625cff88327 100644 --- a/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs +++ b/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs @@ -53,18 +53,18 @@ public IDisposable SubscribeToChanges(Action callback) lock (_subscribers) { _subscribers.Add(subscription); - } - if (Volatile.Read(ref _latest) is { } currentConfig) - { - try + if (_latest is { } currentConfig) { - // If we already have updates, call this immediately - callback(currentConfig); - } - catch (Exception ex) - { - Log.Error(ex, "Error notifying subscriber of updated MutableSettings during subscribe"); + try + { + // If we already have updates, call this immediately + callback(currentConfig); + } + catch (Exception ex) + { + Log.Error(ex, "Error notifying subscriber of updated MutableSettings during subscribe"); + } } } From 684dd635d1a8f09f60c9c2f03e7fcee03a6cd8b4 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Mon, 10 Nov 2025 10:34:39 +0100 Subject: [PATCH 8/9] Apply suggestions from code review Co-authored-by: Lucas Pimentel --- .../src/Datadog.Trace/Configuration/SettingsManager.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs b/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs index 7625cff88327..35b140629d59 100644 --- a/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs +++ b/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs @@ -40,10 +40,10 @@ internal class SettingsManager( /// /// Subscribe to changes in and/or . - /// Called whenever these settings change. If the settings have already changed when - /// is called, is invoked immediately with the latest configuration. + /// is called whenever these settings change. If the settings have already changed when + /// is called, is synchronously invoked immediately with the latest configuration. /// Also note that calling twice with the same callback - /// will invoke the callback twice. + /// will invoke the callback twice. Callbacks should complete quickly to avoid blocking other operations. /// /// The method to invoke /// An that should be disposed to unsubscribe @@ -160,10 +160,9 @@ private void NotifySubscribers(SettingChanges settings) lock (_subscribers) { - var subscribers = _subscribers; Volatile.Write(ref _latest, settings); - foreach (var subscriber in subscribers) + foreach (var subscriber in _subscribers) { try { From 438ebc1e13e7aee2f4682165524921403c35436e Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Mon, 10 Nov 2025 11:10:13 +0000 Subject: [PATCH 9/9] Refactor remote/dynamic config to lock during remote/manual config updates --- .../Tracer/ConfigureIntegration.cs | 7 +-- .../GlobalConfigurationSource.cs | 17 ----- .../DynamicConfigurationManager.cs | 8 +-- .../Configuration/SettingsManager.cs | 62 +++++++++++++------ .../TracerSettingsSettingManagerTests.cs | 5 ++ 5 files changed, 51 insertions(+), 48 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/ConfigureIntegration.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/ConfigureIntegration.cs index 13d9015ddb3a..1f1b6866c535 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/ConfigureIntegration.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/ConfigureIntegration.cs @@ -73,12 +73,7 @@ internal static void ConfigureSettingsWithManualOverrides(Dictionary internal class GlobalConfigurationSource { - private static IConfigurationSource _dynamicConfigConfigurationSource = NullConfigurationSource.Instance; - private static ManualInstrumentationConfigurationSourceBase _manualConfigurationSource = new ManualInstrumentationConfigurationSource(new Dictionary(), useDefaultSources: true); - /// /// Gets the configuration source instance. /// @@ -32,10 +29,6 @@ internal class GlobalConfigurationSource internal static GlobalConfigurationSourceResult CreationResult { get; private set; } = CreateDefaultConfigurationSource(); - internal static IConfigurationSource DynamicConfigurationSource => _dynamicConfigConfigurationSource; - - internal static ManualInstrumentationConfigurationSourceBase ManualConfigurationSource => _manualConfigurationSource; - /// /// Creates a by combining environment variables, /// Precedence is as follows: @@ -142,14 +135,4 @@ private static string GetCurrentDirectory() { return AppDomain.CurrentDomain.BaseDirectory ?? Directory.GetCurrentDirectory(); } - - public static void UpdateDynamicConfigConfigurationSource(IConfigurationSource dynamic) - { - Interlocked.Exchange(ref _dynamicConfigConfigurationSource, dynamic); - } - - public static void UpdateManualConfigurationSource(ManualInstrumentationConfigurationSourceBase manual) - { - Interlocked.Exchange(ref _manualConfigurationSource, manual); - } } diff --git a/tracer/src/Datadog.Trace/Configuration/DynamicConfigurationManager.cs b/tracer/src/Datadog.Trace/Configuration/DynamicConfigurationManager.cs index f046a8293ed0..bcebc7ad13a9 100644 --- a/tracer/src/Datadog.Trace/Configuration/DynamicConfigurationManager.cs +++ b/tracer/src/Datadog.Trace/Configuration/DynamicConfigurationManager.cs @@ -74,13 +74,7 @@ internal static void OnlyForTests_ApplyConfiguration(IConfigurationSource dynami private static void OnConfigurationChanged(IConfigurationSource dynamicConfig, TracerSettings tracerSettings) { - var manualConfig = GlobalConfigurationSource.ManualConfigurationSource; - - // We save this immediately, even if there's no manifest changes in the final settings - // so that it can be picked up by other configuration updaters, e.g. config in code - GlobalConfigurationSource.UpdateDynamicConfigConfigurationSource(dynamicConfig); - - var wasUpdated = tracerSettings.Manager.UpdateSettings(dynamicConfig, manualConfig, TelemetryFactory.Config); + var wasUpdated = tracerSettings.Manager.UpdateDynamicConfigurationSettings(dynamicConfig, TelemetryFactory.Config); if (wasUpdated) { Log.Information("Setting updates made via dynamic configuration were applied"); diff --git a/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs b/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs index 35b140629d59..8a0005207c0a 100644 --- a/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs +++ b/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs @@ -24,6 +24,11 @@ internal class SettingsManager( { private readonly TracerSettings _tracerSettings = tracerSettings; private readonly List _subscribers = []; + + private IConfigurationSource _dynamicConfigurationSource = NullConfigurationSource.Instance; + private ManualInstrumentationConfigurationSourceBase _manualConfigurationSource = + new ManualInstrumentationConfigurationSource(new Dictionary(), useDefaultSources: true); + private SettingChanges? _latest; /// @@ -71,15 +76,44 @@ public IDisposable SubscribeToChanges(Action callback) return subscription; } + /// + /// Regenerate the application's new and + /// based on runtime configuration sources. + /// + /// An containing the new settings created by manual configuration (in code) + /// The central to report config telemetry updates to + /// True if changes were detected and consumers were updated, false otherwise + public bool UpdateManualConfigurationSettings( + ManualInstrumentationConfigurationSourceBase manualSource, + IConfigurationTelemetry centralTelemetry) + { + // we lock this whole method so that we can't conflict with UpdateDynamicConfigurationSettings calls too + lock (_subscribers) + { + _manualConfigurationSource = manualSource; + return UpdateSettings(_dynamicConfigurationSource, manualSource, centralTelemetry); + } + } + /// /// Regenerate the application's new and /// based on runtime configuration sources. /// /// An for dynamic config via remote config - /// An for manual configuration (in code) /// The central to report config telemetry updates to /// True if changes were detected and consumers were updated, false otherwise - public bool UpdateSettings( + public bool UpdateDynamicConfigurationSettings( + IConfigurationSource dynamicConfigSource, + IConfigurationTelemetry centralTelemetry) + { + lock (_subscribers) + { + _dynamicConfigurationSource = dynamicConfigSource; + return UpdateSettings(dynamicConfigSource, _manualConfigurationSource, centralTelemetry); + } + } + + private bool UpdateSettings( IConfigurationSource dynamicConfigSource, ManualInstrumentationConfigurationSourceBase manualSource, IConfigurationTelemetry centralTelemetry) @@ -153,25 +187,17 @@ public bool UpdateSettings( private void NotifySubscribers(SettingChanges settings) { - // Strictly, for safety, we only need to lock in the subscribers list access. However, - // there's nothing to prevent NotifySubscribers being called concurrently, - // which could result in weird out-of-order notifications for customers. So for simplicity - // we just lock the whole method to ensure serialized updates. + _latest = settings; - lock (_subscribers) + foreach (var subscriber in _subscribers) { - Volatile.Write(ref _latest, settings); - - foreach (var subscriber in _subscribers) + try { - try - { - subscriber.Notify(settings); - } - catch (Exception ex) - { - Log.Error(ex, "Error notifying subscriber of MutableSettings change"); - } + subscriber.Notify(settings); + } + catch (Exception ex) + { + Log.Error(ex, "Error notifying subscriber of MutableSettings change"); } } } diff --git a/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsSettingManagerTests.cs b/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsSettingManagerTests.cs index a4fc267033fd..a8dff10efd2c 100644 --- a/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsSettingManagerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsSettingManagerTests.cs @@ -3,9 +3,14 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // +using System.Collections.Generic; +using System.Linq; using Datadog.Trace.Configuration; +using Datadog.Trace.Configuration.ConfigurationSources; +using Datadog.Trace.Configuration.Telemetry; using FluentAssertions; using Xunit; +using SettingChanges = Datadog.Trace.Configuration.TracerSettings.SettingsManager.SettingChanges; namespace Datadog.Trace.Tests.Configuration;