diff --git a/tracer/build/_build/Build.Shared.Steps.cs b/tracer/build/_build/Build.Shared.Steps.cs index 5fab89e0495e..2356e0bfa129 100644 --- a/tracer/build/_build/Build.Shared.Steps.cs +++ b/tracer/build/_build/Build.Shared.Steps.cs @@ -162,7 +162,7 @@ partial class Build { DeleteDirectory(NativeLoaderProject.Directory / "bin"); - var finalArchs = FastDevLoop ? "arm64" : string.Join(';', OsxArchs); + var finalArchs = string.Join(';', OsxArchs); var buildDirectory = NativeBuildDirectory + "_" + finalArchs.Replace(';', '_'); EnsureExistingDirectory(buildDirectory); diff --git a/tracer/build/_build/Build.Steps.cs b/tracer/build/_build/Build.Steps.cs index f4dd25311425..ace11f63433e 100644 --- a/tracer/build/_build/Build.Steps.cs +++ b/tracer/build/_build/Build.Steps.cs @@ -353,7 +353,7 @@ bool RequiresThoroughTesting() { DeleteDirectory(NativeTracerProject.Directory / "build"); - var finalArchs = FastDevLoop ? "arm64" : string.Join(';', OsxArchs); + var finalArchs = string.Join(';', OsxArchs); var buildDirectory = NativeBuildDirectory + "_" + finalArchs.Replace(';', '_'); EnsureExistingDirectory(buildDirectory); diff --git a/tracer/build/_build/Build.Utilities.cs b/tracer/build/_build/Build.Utilities.cs index e7182f52fd7c..6ef27a0d01cf 100644 --- a/tracer/build/_build/Build.Utilities.cs +++ b/tracer/build/_build/Build.Utilities.cs @@ -430,6 +430,11 @@ Target RegenerateSolutions .Description("Regenerates the 'build' solutions based on the 'master' solution") .Executes(() => { + if (FastDevLoop) + { + return; + } + // Create a copy of the "full solution" var sln = ProjectModelTasks.CreateSolution( fileName: RootDirectory / "Datadog.Trace.Samples.g.sln", diff --git a/tracer/build/_build/Build.cs b/tracer/build/_build/Build.cs index 09ebf51f0f0c..cdb6fed7e763 100644 --- a/tracer/build/_build/Build.cs +++ b/tracer/build/_build/Build.cs @@ -142,6 +142,11 @@ public Build() .Description("Cleans all build output") .Executes(() => { + if (FastDevLoop) + { + return; + } + if (IsWin) { // These are created as part of the CreatePlatformlessSymlinks target and cause havok diff --git a/tracer/src/Datadog.Trace.Tools.Runner/CiUtils.cs b/tracer/src/Datadog.Trace.Tools.Runner/CiUtils.cs index d56cf2cd6c4f..033aa219b9cb 100644 --- a/tracer/src/Datadog.Trace.Tools.Runner/CiUtils.cs +++ b/tracer/src/Datadog.Trace.Tools.Runner/CiUtils.cs @@ -269,6 +269,11 @@ async Task UploadRepositoryChangesAsync() profilerEnvironmentVariables[Configuration.ConfigurationKeys.Debugger.ExceptionReplayEnabled] = "1"; profilerEnvironmentVariables[Configuration.ConfigurationKeys.Debugger.RateLimitSeconds] = "0"; profilerEnvironmentVariables[Configuration.ConfigurationKeys.Debugger.UploadFlushInterval] = "1000"; + + if (agentless) + { + profilerEnvironmentVariables[Configuration.ConfigurationKeys.Debugger.ExceptionReplayAgentlessEnabled] = "1"; + } } // Let's set the code coverage datacollector if the code coverage is enabled diff --git a/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.Debugger.cs b/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.Debugger.cs index e4a2481167f0..13ef103aecdc 100644 --- a/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.Debugger.cs +++ b/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.Debugger.cs @@ -179,6 +179,18 @@ internal static class Debugger /// /// public const string CodeOriginMaxUserFrames = "DD_CODE_ORIGIN_FOR_SPANS_MAX_USER_FRAMES"; + + /// + /// Configuration key for enabling or disabling agentless Exception Replay uploads. + /// Default value is false. + /// + public const string ExceptionReplayAgentlessEnabled = "DD_EXCEPTION_REPLAY_AGENTLESS_ENABLED"; + + /// + /// Configuration key for overriding the agentless Exception Replay intake URL. + /// Default value is derived from DD_SITE (https://debugger-intake.<site>/api/v2/debugger). + /// + public const string ExceptionReplayAgentlessUrl = "DD_EXCEPTION_REPLAY_AGENTLESS_URL"; } } } diff --git a/tracer/src/Datadog.Trace/Debugger/DebuggerManager.cs b/tracer/src/Datadog.Trace/Debugger/DebuggerManager.cs index 99ef47f0f23b..648f6102bee5 100644 --- a/tracer/src/Datadog.Trace/Debugger/DebuggerManager.cs +++ b/tracer/src/Datadog.Trace/Debugger/DebuggerManager.cs @@ -200,6 +200,12 @@ private void InitializeSymbolUploaderIfNeeded(TracerSettings tracerSettings, Deb return; } + if (ExceptionReplaySettings.AgentlessEnabled) + { + Log.Information("Exception Replay agentless mode enabled; skipping symbol uploader initialization because it requires the Datadog Agent and Remote Configuration."); + return; + } + if (Interlocked.CompareExchange(ref _symDbInitialized, 1, 0) != 0) { // Once created, the symbol uploader persists even if DI is later disabled diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplay.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplay.cs index 952e32175c47..92af1d044f7a 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplay.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplay.cs @@ -7,13 +7,11 @@ using System; using System.Threading.Tasks; -using Datadog.Trace.Agent; using Datadog.Trace.Debugger.ExceptionAutoInstrumentation.ThirdParty; using Datadog.Trace.Debugger.Helpers; using Datadog.Trace.Debugger.Sink; using Datadog.Trace.Debugger.Snapshots; using Datadog.Trace.Debugger.Upload; -using Datadog.Trace.HttpOverStreams; using Datadog.Trace.Logging; namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation @@ -65,18 +63,26 @@ private void InitSnapshotsSink() // Set up the snapshots sink. var snapshotSlicer = SnapshotSlicer.Create(debuggerSettings); _snapshotSink = SnapshotSink.Create(debuggerSettings, snapshotSlicer); - // TODO: respond to changes in exporter settings - var apiFactory = AgentTransportStrategy.Get( - tracer.Settings.Manager.InitialExporterSettings, - productName: "debugger", - tcpTimeout: TimeSpan.FromSeconds(15), - AgentHttpHeaderNames.MinimalHeaders, - () => new MinimalAgentHeaderHelper(), - uri => uri); var discoveryService = tracer.TracerManager.DiscoveryService; var gitMetadataTagsProvider = tracer.TracerManager.GitMetadataTagsProvider; + ExceptionReplayTransportInfo transportInfo; - var snapshotUploadApi = DebuggerUploadApiFactory.CreateSnapshotUploadApi(apiFactory, discoveryService, gitMetadataTagsProvider); + try + { + transportInfo = ExceptionReplayTransportFactory.Create(tracer.Settings, Settings, discoveryService); + } + catch (InvalidOperationException ex) + { + Log.Error(ex, "Exception Replay transport could not be initialized in agentless mode. Disabling Exception Replay."); + _isDisabled = true; + return; + } + + var snapshotUploadApi = DebuggerUploadApiFactory.CreateSnapshotUploadApi( + transportInfo.ApiRequestFactory, + transportInfo.DiscoveryService, + gitMetadataTagsProvider, + transportInfo.StaticEndpoint); var snapshotBatchUploader = BatchUploader.Create(snapshotUploadApi); _uploader = SnapshotUploader.Create( @@ -84,6 +90,11 @@ private void InitSnapshotsSink() snapshotBatchUploader: snapshotBatchUploader, debuggerSettings); + if (transportInfo.IsAgentless) + { + Log.Information("Exception Replay agentless uploads enabled. Symbol uploads remain unavailable without the Datadog Agent."); + } + _ = Task.Run(() => _uploader.StartFlushingAsync()) .ContinueWith( t => diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplayProbe.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplayProbe.cs index 3f6fc5b94b49..077a7f4c8714 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplayProbe.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplayProbe.cs @@ -5,19 +5,23 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using Datadog.Trace.Debugger.Expressions; using Datadog.Trace.Debugger.Helpers; +using Datadog.Trace.Debugger.PInvoke; using Datadog.Trace.Debugger.RateLimiting; using Datadog.Trace.Debugger.Sink.Models; -using Datadog.Trace.Vendors.Serilog; +using Datadog.Trace.Logging; +using Datadog.Trace.Vendors.Serilog.Events; #nullable enable namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation { internal class ExceptionReplayProbe { + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); private readonly int _hashCode; private readonly object _locker = new(); private readonly List _exceptionCases = new(); @@ -107,6 +111,8 @@ private void ProcessCase(ExceptionCase @case) internal void AddExceptionCase(ExceptionCase @case, bool isPartOfCase) { + var shouldRefreshAfterLock = false; + lock (_locker) { if (isPartOfCase && ShouldInstrument()) @@ -134,6 +140,13 @@ internal void AddExceptionCase(ExceptionCase @case, bool isPartOfCase) _exceptionCases.Add(@case); ProcessCase(@case); + + shouldRefreshAfterLock = @case.Probes?.Length == 1; + } + + if (shouldRefreshAfterLock) + { + TryRefreshSingleFrameProbeStatus(); } } @@ -166,5 +179,85 @@ public override int GetHashCode() { return _hashCode; } + + /// + /// If an exception case only contains a single customer frame, we never build parent/child call-path hashes, + /// meaning the ordinary probe-status polling code in never executes. + /// For CI Visibility (and other single-frame scenarios) this left probes permanently stuck in the default + /// state, so snapshots were never captured. To avoid changing the behaviour + /// for multi-frame cases, we perform a one-off eager poll right after the probe is attached. The poll is + /// executed outside the probe lock because we may wait up to a few seconds while the CLR completes ReJIT and + /// we do not want to block unrelated instrumentation updates. + /// + private void TryRefreshSingleFrameProbeStatus() + { + // Only apply this patch to the test optimization product + if (!Datadog.Trace.Ci.TestOptimization.Instance.IsRunning) + { + return; + } + + if (string.IsNullOrEmpty(ProbeId)) + { + return; + } + + try + { + // In practice the native tracer reports INSTALLED for ~500 ms after we request ReJIT, but CI Visibility + // tests regularly need a little longer (module load + async offloader). We therefore try a handful of + // times with a generous delay so we can observe the final INSTRUMENTED status without changing the + // behaviour for other scenarios. + const int maxAttempts = 20; + var stopwatch = Stopwatch.StartNew(); + + for (var attempt = 0; attempt < maxAttempts; attempt++) + { + var statuses = DebuggerNativeMethods.GetProbesStatuses(new[] { ProbeId }); + if (statuses.Length == 0) + { + return; + } + + var previous = ProbeStatus; + ProbeStatus = statuses[0].Status; + ErrorMessage = statuses[0].ErrorMessage; + + if (Log.IsEnabled(LogEventLevel.Debug)) + { + var message = $"Eager status refresh for single-frame probe {ProbeId}. Previous={previous}, Current={ProbeStatus}, Attempt={attempt + 1}, ElapsedMs={stopwatch.ElapsedMilliseconds}"; + Log.Debug("{Message}", message); + } + + if (ProbeStatus == Status.INSTRUMENTED) + { + break; + } + + if (ProbeStatus == Status.ERROR || ProbeStatus == Status.BLOCKED) + { + break; + } + + if (attempt < maxAttempts - 1) + { + Thread.Sleep(attempt == 0 ? 1_500 : 250); + } + } + + if (ProbeStatus != Status.INSTRUMENTED) + { + Log.Warning( + "Single-frame probe {ProbeId} never reported INSTRUMENTED during eager refresh. FinalStatus={Status}, TotalWaitMs={ElapsedMs}", + ProbeId, + ProbeStatus, + stopwatch.ElapsedMilliseconds); + } + } + catch (Exception ex) + { + Log.Warning(ex, "Failed to eagerly refresh probe status for {ProbeId}", ProbeId); + } + } } } diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplaySettings.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplaySettings.cs index c9681749780b..92c5ba6430d6 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplaySettings.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplaySettings.cs @@ -16,6 +16,7 @@ internal class ExceptionReplaySettings public const int DefaultMaxFramesToCapture = 4; public const int DefaultRateLimitSeconds = 60 * 60; // 1 hour public const int DefaultMaxExceptionAnalysisLimit = 100; + private const string DefaultSite = "datadoghq.com"; public ExceptionReplaySettings(IConfigurationSource? source, IConfigurationTelemetry telemetry) { @@ -47,6 +48,11 @@ public ExceptionReplaySettings(IConfigurationSource? source, IConfigurationTelem .WithKeys(ConfigurationKeys.Debugger.MaxExceptionAnalysisLimit) .AsInt32(DefaultMaxExceptionAnalysisLimit, x => x > 0) .Value; + + AgentlessEnabled = config.WithKeys(ConfigurationKeys.Debugger.ExceptionReplayAgentlessEnabled).AsBool(false); + AgentlessUrlOverride = config.WithKeys(ConfigurationKeys.Debugger.ExceptionReplayAgentlessUrl).AsString(); + AgentlessApiKey = config.WithKeys(ConfigurationKeys.ApiKey).AsRedactedString(); + AgentlessSite = config.WithKeys(ConfigurationKeys.Site).AsString(DefaultSite, site => !string.IsNullOrEmpty(site)) ?? DefaultSite; } public bool Enabled { get; } @@ -61,6 +67,14 @@ public ExceptionReplaySettings(IConfigurationSource? source, IConfigurationTelem public int MaxExceptionAnalysisLimit { get; } + public bool AgentlessEnabled { get; } + + public string? AgentlessUrlOverride { get; } + + public string? AgentlessApiKey { get; } + + public string AgentlessSite { get; } + public static ExceptionReplaySettings FromSource(IConfigurationSource source, IConfigurationTelemetry telemetry) { return new ExceptionReplaySettings(source, telemetry); diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplayTransportFactory.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplayTransportFactory.cs new file mode 100644 index 000000000000..6a6a583bd064 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplayTransportFactory.cs @@ -0,0 +1,121 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Generic; +using Datadog.Trace.Agent; +using Datadog.Trace.Agent.DiscoveryService; +using Datadog.Trace.Configuration; +using Datadog.Trace.Debugger; +using Datadog.Trace.HttpOverStreams; +using Datadog.Trace.Logging; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal static class ExceptionReplayTransportFactory + { + private const string DefaultRelativePath = "/api/v2/debugger"; + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(ExceptionReplayTransportFactory)); + + internal static ExceptionReplayTransportInfo Create(TracerSettings tracerSettings, ExceptionReplaySettings settings, IDiscoveryService discoveryService) + { + if (!settings.AgentlessEnabled) + { + return CreateAgentTransport(tracerSettings, discoveryService); + } + + if (string.IsNullOrWhiteSpace(settings.AgentlessApiKey)) + { + Log.Error("Exception Replay agentless uploads enabled but DD_API_KEY is not set. Disabling Exception Replay."); + throw new InvalidOperationException("Exception Replay agentless mode requires DD_API_KEY."); + } + + if (!TryResolveAgentlessEndpoint(settings, out var baseUri, out var relativePath)) + { + Log.Error("Exception Replay agentless uploads enabled but a valid intake URL could not be determined. Disabling Exception Replay."); + throw new InvalidOperationException("Exception Replay agentless mode requires a valid intake URL."); + } + + var apiFactory = DebuggerTransportStrategy.Get(baseUri); + apiFactory = new HeaderInjectingApiRequestFactory(apiFactory, settings.AgentlessApiKey!); + return new ExceptionReplayTransportInfo(apiFactory, null, relativePath, isAgentless: true); + } + + private static ExceptionReplayTransportInfo CreateAgentTransport(TracerSettings tracerSettings, IDiscoveryService discoveryService) + { + var apiFactory = Agent.AgentTransportStrategy.Get( + tracerSettings.Manager.InitialExporterSettings, + productName: "exception-replay", + tcpTimeout: TimeSpan.FromSeconds(15), + AgentHttpHeaderNames.MinimalHeaders, + () => new MinimalAgentHeaderHelper(), + uri => uri); + + return new ExceptionReplayTransportInfo(apiFactory, discoveryService, staticEndpoint: null, isAgentless: false); + } + + private static bool TryResolveAgentlessEndpoint(ExceptionReplaySettings settings, out Uri baseUri, out string relativePath) + { + var overrideUrl = settings.AgentlessUrlOverride; + if (string.IsNullOrWhiteSpace(overrideUrl)) + { + baseUri = new Uri($"https://debugger-intake.{settings.AgentlessSite}/"); + relativePath = DefaultRelativePath; + return true; + } + + if (!Uri.TryCreate(overrideUrl, UriKind.Absolute, out var uri)) + { + baseUri = null!; + relativePath = string.Empty; + return false; + } + + baseUri = new Uri($"{uri.Scheme}://{uri.Authority}/"); + var path = uri.PathAndQuery; + relativePath = string.IsNullOrEmpty(path) ? DefaultRelativePath : EnsureLeadingSlash(path); + return true; + } + + private static string EnsureLeadingSlash(string value) + { + return value.StartsWith("/", StringComparison.Ordinal) ? value : "/" + value; + } + + private sealed class HeaderInjectingApiRequestFactory : IApiRequestFactory + { + private const string ApiKeyHeader = "DD-API-KEY"; + private const string RequestIdHeader = "DD-REQUEST-ID"; + private const string EvpOriginHeader = "DD-EVP-ORIGIN"; + private const string OriginValue = "dd-trace-dotnet"; + + private readonly IApiRequestFactory _inner; + private readonly string _apiKey; + + public HeaderInjectingApiRequestFactory(IApiRequestFactory inner, string apiKey) + { + _inner = inner; + _apiKey = apiKey; + } + + public string Info(Uri endpoint) => _inner.Info(endpoint); + + public Uri GetEndpoint(string relativePath) => _inner.GetEndpoint(relativePath); + + public IApiRequest Create(Uri endpoint) + { + var request = _inner.Create(endpoint); + request.AddHeader(ApiKeyHeader, _apiKey); + request.AddHeader(EvpOriginHeader, OriginValue); + request.AddHeader(RequestIdHeader, Guid.NewGuid().ToString()); + return request; + } + + public void SetProxy(System.Net.WebProxy proxy, System.Net.NetworkCredential credential) => _inner.SetProxy(proxy, credential); + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplayTransportInfo.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplayTransportInfo.cs new file mode 100644 index 000000000000..17a74c9eb784 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplayTransportInfo.cs @@ -0,0 +1,31 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using Datadog.Trace.Agent; +using Datadog.Trace.Agent.DiscoveryService; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal readonly struct ExceptionReplayTransportInfo + { + public ExceptionReplayTransportInfo(IApiRequestFactory apiRequestFactory, IDiscoveryService? discoveryService, string? staticEndpoint, bool isAgentless) + { + ApiRequestFactory = apiRequestFactory; + DiscoveryService = discoveryService; + StaticEndpoint = staticEndpoint; + IsAgentless = isAgentless; + } + + public IApiRequestFactory ApiRequestFactory { get; } + + public IDiscoveryService? DiscoveryService { get; } + + public string? StaticEndpoint { get; } + + public bool IsAgentless { get; } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/Upload/DebuggerUploadApiFactory.cs b/tracer/src/Datadog.Trace/Debugger/Upload/DebuggerUploadApiFactory.cs index c09cc50b24ea..a81cabb79731 100644 --- a/tracer/src/Datadog.Trace/Debugger/Upload/DebuggerUploadApiFactory.cs +++ b/tracer/src/Datadog.Trace/Debugger/Upload/DebuggerUploadApiFactory.cs @@ -12,9 +12,9 @@ namespace Datadog.Trace.Debugger.Upload { internal static class DebuggerUploadApiFactory { - internal static IBatchUploadApi CreateSnapshotUploadApi(IApiRequestFactory apiRequestFactory, IDiscoveryService discoveryService, IGitMetadataTagsProvider gitMetadataTagsProvider) + internal static IBatchUploadApi CreateSnapshotUploadApi(IApiRequestFactory apiRequestFactory, IDiscoveryService? discoveryService, IGitMetadataTagsProvider gitMetadataTagsProvider, string? staticEndpoint = null) { - return SnapshotUploadApi.Create(apiRequestFactory, discoveryService, gitMetadataTagsProvider); + return SnapshotUploadApi.Create(apiRequestFactory, discoveryService, gitMetadataTagsProvider, staticEndpoint); } internal static IBatchUploadApi CreateLogUploadApi(IApiRequestFactory apiRequestFactory, IDiscoveryService discoveryService, IGitMetadataTagsProvider gitMetadataTagsProvider) diff --git a/tracer/src/Datadog.Trace/Debugger/Upload/SnapshotUploadApi.cs b/tracer/src/Datadog.Trace/Debugger/Upload/SnapshotUploadApi.cs index 11db85ece1fa..d1795809e73a 100644 --- a/tracer/src/Datadog.Trace/Debugger/Upload/SnapshotUploadApi.cs +++ b/tracer/src/Datadog.Trace/Debugger/Upload/SnapshotUploadApi.cs @@ -19,23 +19,36 @@ internal class SnapshotUploadApi : DebuggerUploadApiBase private SnapshotUploadApi( IApiRequestFactory apiRequestFactory, - IDiscoveryService discoveryService, - IGitMetadataTagsProvider gitMetadataTagsProvider) + IDiscoveryService? discoveryService, + IGitMetadataTagsProvider gitMetadataTagsProvider, + string? staticEndpoint) : base(apiRequestFactory, gitMetadataTagsProvider) { - discoveryService.SubscribeToChanges(c => + if (!string.IsNullOrEmpty(staticEndpoint)) { - Endpoint = c.DebuggerV2Endpoint ?? c.DiagnosticsEndpoint; - Log.Debug("SnapshotUploadApi: Updated endpoint to {Endpoint}", Endpoint); - }); + Endpoint = staticEndpoint; + } + else if (discoveryService is not null) + { + discoveryService.SubscribeToChanges(c => + { + Endpoint = c.DebuggerV2Endpoint ?? c.DiagnosticsEndpoint; + Log.Debug("SnapshotUploadApi: Updated endpoint to {Endpoint}", Endpoint); + }); + } + else + { + Log.Warning("SnapshotUploadApi: No discovery service or static endpoint available. Snapshots will not be uploaded until an endpoint is configured."); + } } public static SnapshotUploadApi Create( IApiRequestFactory apiRequestFactory, - IDiscoveryService discoveryService, - IGitMetadataTagsProvider gitMetadataTagsProvider) + IDiscoveryService? discoveryService, + IGitMetadataTagsProvider gitMetadataTagsProvider, + string? staticEndpoint) { - return new SnapshotUploadApi(apiRequestFactory, discoveryService, gitMetadataTagsProvider); + return new SnapshotUploadApi(apiRequestFactory, discoveryService, gitMetadataTagsProvider, staticEndpoint); } public override async Task SendBatchAsync(ArraySegment data) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkRetriesTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkRetriesTests.cs index 194acd831557..6c8e5840095d 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkRetriesTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkRetriesTests.cs @@ -150,6 +150,7 @@ public virtual async Task FlakyRetriesWithExceptionReplay(string packageVersion) // AlwaysFails => 1 + 5 retries var alwaysFailsTests = tests.Where(t => t.Resource == AlwaysFails).ToList(); alwaysFailsTests.Should().Contain(t => t.Meta[TestTags.Status] == TestTags.StatusFail); + alwaysFailsTests = alwaysFailsTests.Where(t => t.Meta.ContainsKey("error.stack")).ToList(); CheckForEnoughNumberOfStackFrames(alwaysFailsTests); alwaysFailsTests.Should().Contain(t => t.Meta.ContainsKey("_dd.di._er")); alwaysFailsTests.Should().Contain(t => t.Meta.ContainsKey("_dd.di._eh")); @@ -171,6 +172,7 @@ public virtual async Task FlakyRetriesWithExceptionReplay(string packageVersion) var trueAtLastRetryTests = tests.Where(t => t.Resource == TrueAtLastRetry).ToList(); trueAtLastRetryTests.Should().Contain(t => t.Meta[TestTags.Status] == TestTags.StatusPass); trueAtLastRetryTests.Should().Contain(t => t.Meta[TestTags.Status] == TestTags.StatusFail); + trueAtLastRetryTests = trueAtLastRetryTests.Where(t => t.Meta.ContainsKey("error.stack")).ToList(); CheckForEnoughNumberOfStackFrames(trueAtLastRetryTests); trueAtLastRetryTests.Should().Contain(t => t.Meta.ContainsKey("_dd.di._er")); trueAtLastRetryTests.Should().Contain(t => t.Meta.ContainsKey("_dd.di._eh")); @@ -183,6 +185,7 @@ public virtual async Task FlakyRetriesWithExceptionReplay(string packageVersion) trueAtThirdRetryTests.Should().HaveCount(1 + 3); trueAtThirdRetryTests.Should().Contain(t => t.Meta[TestTags.Status] == TestTags.StatusPass); trueAtThirdRetryTests.Should().Contain(t => t.Meta[TestTags.Status] == TestTags.StatusFail); + trueAtThirdRetryTests = trueAtThirdRetryTests.Where(t => t.Meta.ContainsKey("error.stack")).ToList(); CheckForEnoughNumberOfStackFrames(trueAtThirdRetryTests); trueAtThirdRetryTests.Should().Contain(t => t.Meta.ContainsKey("_dd.di._er")); trueAtThirdRetryTests.Should().Contain(t => t.Meta.ContainsKey("_dd.di._eh")); @@ -200,11 +203,11 @@ public virtual async Task FlakyRetriesWithExceptionReplay(string packageVersion) private static void CheckForEnoughNumberOfStackFrames(List tests) { Skip.If( - tests.Any(t => NumberOfOccurrences(t.Meta["error.stack"], "\\n at ") == 1), + tests.Any(t => NumberOfOccurrences(t.Meta["error.stack"], " at ") == 1), "There are stacktraces with only 1 stackframe, these kind of exception doesn't have any debugger info because that stack frame always refers to the throw frame."); Skip.If( - tests.Any(t => NumberOfOccurrences(t.Meta["error.stack"], "\\n at ") == NumberOfOccurrences(t.Meta["error.stack"], "\\n at Xunit.")), + tests.Any(t => NumberOfOccurrences(t.Meta["error.stack"], " at ") == NumberOfOccurrences(t.Meta["error.stack"], " at Xunit.")), "All stackframes in the exception contains information of the Xunit assembly only."); } diff --git a/tracer/test/Datadog.Trace.Tests/Debugger/ExceptionReplaySettingsTests.cs b/tracer/test/Datadog.Trace.Tests/Debugger/ExceptionReplaySettingsTests.cs new file mode 100644 index 000000000000..861736697382 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Debugger/ExceptionReplaySettingsTests.cs @@ -0,0 +1,49 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System.Collections.Specialized; +using Datadog.Trace.Configuration; +using Datadog.Trace.Configuration.Telemetry; +using Datadog.Trace.Debugger.ExceptionAutoInstrumentation; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.Debugger +{ + public class ExceptionReplaySettingsTests + { + [Fact] + public void AgentlessSettings_Defaults() + { + var settings = new ExceptionReplaySettings(new NameValueConfigurationSource(new NameValueCollection()), NullConfigurationTelemetry.Instance); + + settings.AgentlessEnabled.Should().BeFalse(); + settings.AgentlessApiKey.Should().BeNull(); + settings.AgentlessUrlOverride.Should().BeNull(); + settings.AgentlessSite.Should().Be("datadoghq.com"); + } + + [Fact] + public void AgentlessSettings_CustomValues() + { + var collection = new NameValueCollection + { + { ConfigurationKeys.Debugger.ExceptionReplayAgentlessEnabled, "true" }, + { ConfigurationKeys.Debugger.ExceptionReplayAgentlessUrl, "https://example.com/custom" }, + { ConfigurationKeys.ApiKey, "test-key" }, + { ConfigurationKeys.Site, "datadoghq.eu" } + }; + + var settings = new ExceptionReplaySettings(new NameValueConfigurationSource(collection), NullConfigurationTelemetry.Instance); + + settings.AgentlessEnabled.Should().BeTrue(); + settings.AgentlessApiKey.Should().Be("test-key"); + settings.AgentlessUrlOverride.Should().Be("https://example.com/custom"); + settings.AgentlessSite.Should().Be("datadoghq.eu"); + } + } +} diff --git a/tracer/test/Datadog.Trace.Tests/Debugger/ExceptionReplayTransportFactoryTests.cs b/tracer/test/Datadog.Trace.Tests/Debugger/ExceptionReplayTransportFactoryTests.cs new file mode 100644 index 000000000000..611832fdb549 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Debugger/ExceptionReplayTransportFactoryTests.cs @@ -0,0 +1,165 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; +using Datadog.Trace.Agent; +using Datadog.Trace.Agent.DiscoveryService; +using Datadog.Trace.Agent.Transports; +using Datadog.Trace.Configuration; +using Datadog.Trace.Configuration.Telemetry; +using Datadog.Trace.Debugger.ExceptionAutoInstrumentation; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.Debugger +{ + public class ExceptionReplayTransportFactoryTests + { + [Fact] + public void ReturnsAgentTransport_WhenAgentlessDisabled() + { + var tracerSettings = new TracerSettings(new NameValueConfigurationSource(new NameValueCollection())); + var erSettings = new ExceptionReplaySettings(new NameValueConfigurationSource(new NameValueCollection()), NullConfigurationTelemetry.Instance); + var discovery = new TestDiscoveryService(); + + var transport = ExceptionReplayTransportFactory.Create(tracerSettings, erSettings, discovery); + + transport.IsAgentless.Should().BeFalse(); + transport.DiscoveryService.Should().Be(discovery); + transport.StaticEndpoint.Should().BeNull(); + } + + [Fact] + public void ReturnsAgentlessTransport_WhenConfigured() + { + var tracerSettings = new TracerSettings(new NameValueConfigurationSource(new NameValueCollection())); + var collection = new NameValueCollection + { + { ConfigurationKeys.Debugger.ExceptionReplayAgentlessEnabled, "true" }, + { ConfigurationKeys.ApiKey, "test-key" } + }; + var erSettings = new ExceptionReplaySettings(new NameValueConfigurationSource(collection), NullConfigurationTelemetry.Instance); + var discovery = new TestDiscoveryService(); + + var transport = ExceptionReplayTransportFactory.Create(tracerSettings, erSettings, discovery); + + transport.IsAgentless.Should().BeTrue(); + transport.DiscoveryService.Should().BeNull(); + transport.StaticEndpoint.Should().Be("/api/v2/debugger"); + } + + [Fact] + public void AgentlessTransport_UsesOverrideUrl() + { + var tracerSettings = new TracerSettings(new NameValueConfigurationSource(new NameValueCollection())); + var collection = new NameValueCollection + { + { ConfigurationKeys.Debugger.ExceptionReplayAgentlessEnabled, "true" }, + { ConfigurationKeys.ApiKey, "test-key" }, + { ConfigurationKeys.Debugger.ExceptionReplayAgentlessUrl, "https://custom-host.example.com/custom/path" } + }; + var erSettings = new ExceptionReplaySettings(new NameValueConfigurationSource(collection), NullConfigurationTelemetry.Instance); + var discovery = new TestDiscoveryService(); + + var transport = ExceptionReplayTransportFactory.Create(tracerSettings, erSettings, discovery); + + transport.IsAgentless.Should().BeTrue(); + transport.StaticEndpoint.Should().Be("/custom/path"); + transport.ApiRequestFactory.GetEndpoint("/api/v2/debugger") + .ToString() + .Should().Be("https://custom-host.example.com/api/v2/debugger"); + } + + [Fact] + public void HeaderInjectingFactory_AddsDebuggerHeaders() + { + var nestedType = typeof(ExceptionReplayTransportFactory) + .GetNestedType("HeaderInjectingApiRequestFactory", BindingFlags.NonPublic); + nestedType.Should().NotBeNull(); + var ctor = nestedType!.GetConstructor( + BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public, + binder: null, + new[] { typeof(IApiRequestFactory), typeof(string) }, + modifiers: null); + ctor.Should().NotBeNull(); + + var recordingFactory = new RecordingApiRequestFactory(); + var wrapper = (IApiRequestFactory)ctor!.Invoke(new object[] { recordingFactory, "secret-key" }); + + wrapper.Create(new Uri("https://debugger-intake.example.com/api/v2/debugger")); + + var recordedRequest = recordingFactory.Requests.Single(); + recordedRequest.Headers.Should().ContainKey("DD-API-KEY").WhoseValue.Should().Be("secret-key"); + recordedRequest.Headers.Should().ContainKey("DD-EVP-ORIGIN").WhoseValue.Should().Be("dd-trace-dotnet"); + recordedRequest.Headers.Should().ContainKey("DD-REQUEST-ID"); + Guid.TryParse(recordedRequest.Headers["DD-REQUEST-ID"], out _).Should().BeTrue(); + } + + private sealed class TestDiscoveryService : IDiscoveryService + { + public void SubscribeToChanges(System.Action callback) + { + } + + public void RemoveSubscription(System.Action callback) + { + } + + public Task DisposeAsync() => Task.CompletedTask; + } + + private sealed class RecordingApiRequestFactory : IApiRequestFactory + { + public List Requests { get; } = new(); + + public string Info(Uri endpoint) => endpoint.ToString(); + + public Uri GetEndpoint(string relativePath) => new($"https://placeholder{relativePath}"); + + public IApiRequest Create(Uri endpoint) + { + var request = new RecordingApiRequest(endpoint); + Requests.Add(request); + return request; + } + + public void SetProxy(System.Net.WebProxy proxy, System.Net.NetworkCredential credential) + { + } + } + + private sealed class RecordingApiRequest : IApiRequest + { + public RecordingApiRequest(Uri endpoint) + { + Endpoint = endpoint; + } + + public Uri Endpoint { get; } + + public Dictionary Headers { get; } = new(); + + public void AddHeader(string name, string value) => Headers[name] = value; + + public Task GetAsync() => throw new NotSupportedException(); + + public Task PostAsync(ArraySegment bytes, string contentType) => throw new NotSupportedException(); + + public Task PostAsync(ArraySegment bytes, string contentType, string contentEncoding) => throw new NotSupportedException(); + + public Task PostAsync(Func writeToRequestStream, string contentType, string contentEncoding, string multipartBoundary) => throw new NotSupportedException(); + + public Task PostAsync(MultipartFormItem[] items, MultipartCompression multipartCompression = MultipartCompression.None) => throw new NotSupportedException(); + } + } +} diff --git a/tracer/test/Datadog.Trace.Tests/Debugger/SnapshotUploadApiTests.cs b/tracer/test/Datadog.Trace.Tests/Debugger/SnapshotUploadApiTests.cs new file mode 100644 index 000000000000..e2ddb9d1e338 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Debugger/SnapshotUploadApiTests.cs @@ -0,0 +1,37 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Threading.Tasks; +using Datadog.Trace.Configuration; +using Datadog.Trace.Debugger.Upload; +using Datadog.Trace.TestHelpers.TransportHelpers; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.Debugger +{ + public class SnapshotUploadApiTests + { + [Fact] + public async Task UsesStaticEndpointWhenProvided() + { + var factory = new TestRequestFactory(new Uri("https://debugger-intake.example/")); + var gitMetadataProvider = new NullGitMetadataProvider(); + var snapshotApi = SnapshotUploadApi.Create(factory, discoveryService: null, gitMetadataProvider, staticEndpoint: "/api/v2/debugger"); + + var payload = new ArraySegment(new byte[] { 1, 2, 3 }); + var success = await snapshotApi.SendBatchAsync(payload); + + success.Should().BeTrue(); + factory.RequestsSent.Should().ContainSingle(); + var requestUri = factory.RequestsSent[0].Endpoint; + requestUri.GetLeftPart(UriPartial.Path).Should().Be("https://debugger-intake.example/api/v2/debugger"); + requestUri.Query.Should().Contain("ddtags="); + } + } +} diff --git a/tracer/test/Datadog.Trace.Tests/Telemetry/config_norm_rules.json b/tracer/test/Datadog.Trace.Tests/Telemetry/config_norm_rules.json index 3793c7268d30..84161c1b2129 100644 --- a/tracer/test/Datadog.Trace.Tests/Telemetry/config_norm_rules.json +++ b/tracer/test/Datadog.Trace.Tests/Telemetry/config_norm_rules.json @@ -429,6 +429,8 @@ "DD_EXCEPTION_DEBUGGING_CAPTURE_FULL_CALLSTACK_ENABLED": "dd_exception_debugging_capture_full_callstack_enabled", "DD_EXCEPTION_DEBUGGING_RATE_LIMIT_SECONDS": "dd_exception_debugging_rate_limit_seconds", "DD_EXCEPTION_DEBUGGING_MAX_EXCEPTION_ANALYSIS_LIMIT": "dd_exception_debugging_max_exception_analysis_limit", + "DD_EXCEPTION_REPLAY_AGENTLESS_ENABLED": "dd_exception_replay_agentless_enabled", + "DD_EXCEPTION_REPLAY_AGENTLESS_URL": "dd_exception_replay_agentless_url", "DD_EXCEPTION_REPLAY_ENABLED": "dd_exception_replay_enabled", "DD_EXCEPTION_REPLAY_MAX_FRAMES_TO_CAPTURE": "dd_exception_replay_max_frames_to_capture", "DD_EXCEPTION_REPLAY_CAPTURE_FULL_CALLSTACK_ENABLED": "dd_exception_replay_capture_full_callstack_enabled",