[WIP] [Azure Functions] Fix span parenting for isolated functions with ASP.NET Core integration

tracer/src/Datadog.AzureFunctions/Datadog.AzureFunctions.csproj

@@ -27,8 +27,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <ProjectReference Include="$(MSBuildThisFileDirectory)..\Datadog.Trace.Manual\Datadog.Trace.Manual.csproj" />
-    <ProjectReference Include="$(MSBuildThisFileDirectory)..\Datadog.Trace.Annotations\Datadog.Trace.Annotations.csproj" />
+    <PackageReference Include="Datadog.Trace" Version="*"/>
+    <PackageReference Include="Datadog.Trace.Annotations" Version="*" />
 
     <!-- Use PrivateAssets="none" to stop excluding content files by default -->
     <PackageReference Include="Datadog.Serverless.Compat" Version="1.1.0" PrivateAssets="none" />

tracer/src/Datadog.Trace/Agent/MessagePack/SpanMessagePackFormatter.cs

@@ -112,6 +112,10 @@ internal class SpanMessagePackFormatter : IMessagePackFormatter<TraceChunkModel>
         private byte[] _aasRuntimeTagNameBytes;
         private byte[] _aasExtensionVersionTagNameBytes;
 
+        // Azure Functions tag names and values
+        private byte[] _azureFunctionProcessTagNameBytes;
+        private byte[] _azureFunctionProcessTagValueBytes;
+
         private SpanMessagePackFormatter()
         {
         }
@@ -737,7 +741,7 @@ private int WriteTags(ref byte[] bytes, int offset, in SpanModel model, ITagProc
                 }
 
                 tagBytes = MessagePackStringCache.GetAzureAppServiceKeyBytes(Datadog.Trace.Tags.AzureAppServicesSiteName, azureAppServiceSettings.SiteName);
-                // the front-end identify AAS spans using aas.site.name and aas.site.type, so we need them on all spans
+                // the front-end identifies AAS spans using aas.site.name and aas.site.type, so we need them on all spans
                 if (tagBytes is not null)
                 {
                     count++;
@@ -752,6 +756,16 @@ private int WriteTags(ref byte[] bytes, int offset, in SpanModel model, ITagProc
                     offset += MessagePackBinary.WriteStringBytes(ref bytes, offset, _aasSiteTypeTagNameBytes);
                     offset += MessagePackBinary.WriteRaw(ref bytes, offset, tagBytes);
                 }
+
+                // Add function process tag ("host" or "worker") for isolated Azure Functions
+                if (azureAppServiceSettings.IsIsolatedFunctionsApp &&
+                    _azureFunctionProcessTagNameBytes is not null &&
+                    _azureFunctionProcessTagValueBytes is not null)
+                {
+                    count++;
+                    offset += MessagePackBinary.WriteStringBytes(ref bytes, offset, _azureFunctionProcessTagNameBytes);
+                    offset += MessagePackBinary.WriteStringBytes(ref bytes, offset, _azureFunctionProcessTagValueBytes);
+                }
             }
 
             if (count > 0)
@@ -977,6 +991,15 @@ private void InitializeAasTags()
                 _aasOperatingSystemTagNameBytes = StringEncoding.UTF8.GetBytes(Datadog.Trace.Tags.AzureAppServicesOperatingSystem);
                 _aasRuntimeTagNameBytes = StringEncoding.UTF8.GetBytes(Datadog.Trace.Tags.AzureAppServicesRuntime);
                 _aasExtensionVersionTagNameBytes = StringEncoding.UTF8.GetBytes(Datadog.Trace.Tags.AzureAppServicesExtensionVersion);
+
+                if (EnvironmentHelpers.IsAzureFunctionsIsolated())
+                {
+                    _azureFunctionProcessTagNameBytes = StringEncoding.UTF8.GetBytes(Datadog.Trace.Tags.AzureFunctionProcess);
+
+                    _azureFunctionProcessTagValueBytes = EnvironmentHelpers.IsRunningInAzureFunctionsHost() ?
+                                                             StringEncoding.UTF8.GetBytes("host") :
+                                                             StringEncoding.UTF8.GetBytes("worker");
+                }
             }
         }
 

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IFunctionContext.cs

@@ -18,6 +18,8 @@ internal interface IFunctionContext
     FunctionDefinitionStruct FunctionDefinition { get; }
 
     IEnumerable<KeyValuePair<Type, object?>>? Features { get; }
+
+    IDictionary<object, object>? Items { get; }
 }
 
 #endif

tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs

@@ -470,20 +470,25 @@ private static void StartDiagnosticManager()
         {
             var observers = new List<DiagnosticObserver>();
 
-            // get environment variables directly so we don't access Trace.Instance yet
-            var functionsExtensionVersion = EnvironmentHelpers.GetEnvironmentVariable(PlatformKeys.AzureFunctions.FunctionsExtensionVersion);
-            var functionsWorkerRuntime = EnvironmentHelpers.GetEnvironmentVariable(PlatformKeys.AzureFunctions.FunctionsWorkerRuntime);
-
-            if (!string.IsNullOrEmpty(functionsExtensionVersion) && !string.IsNullOrEmpty(functionsWorkerRuntime))
-            {
-                // Not adding the `AspNetCoreDiagnosticObserver` is particularly important for in-process Azure Functions.
-                // The AspNetCoreDiagnosticObserver will be loaded in a separate Assembly Load Context, breaking the connection of AsyncLocal.
-                // This is because user code is loaded within the functions host in a separate context.
-                // Even in isolated functions, we don't want the AspNetCore spans to be created.
-                Log.Debug("Skipping AspNetCoreDiagnosticObserver in Azure Functions.");
+            // For Azure Functions, we need to handle AspNetCoreDiagnosticObserver differently based on the process type.
+            // Skip AspNetCoreDiagnosticObserver in:
+            // - In-process functions (due to separate Assembly Load Context issues)
+            // - Isolated functions host process (to avoid duplicate spans)
+            // Enable AspNetCoreDiagnosticObserver in:
+            // - Isolated functions worker process (to create aspnet_core.request spans that azure_functions.invoke can parent to)
+            // - All other scenarios (non-Azure Functions)
+            var isAzureFunctionsIsolatedHost = EnvironmentHelpers.IsRunningInAzureFunctionsHost();
+            var isAzureFunctionsInProcess = EnvironmentHelpers.IsAzureFunctions() && !EnvironmentHelpers.IsAzureFunctionsIsolated();
+            var shouldSkipAspNetCore = isAzureFunctionsIsolatedHost || isAzureFunctionsInProcess;
+
+            if (shouldSkipAspNetCore)
+            {
+                // Skip AspNetCoreDiagnosticObserver in Azure Functions host process or in-process functions
+                Log.Debug("Skipping AspNetCoreDiagnosticObserver in Azure Functions (host process or in-process).");
             }
             else
             {
+                // Enable AspNetCoreDiagnosticObserver in isolated worker process or other scenarios
                 observers.Add(new AspNetCoreDiagnosticObserver());
                 observers.Add(new QuartzDiagnosticObserver());
             }

tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/TagListGenerator/AzureFunctionsTags.g.cs

@@ -26,6 +26,10 @@ partial class AzureFunctionsTags
         private static ReadOnlySpan<byte> BindingSourceBytes => new byte[] { 180, 97, 97, 115, 46, 102, 117, 110, 99, 116, 105, 111, 110, 46, 98, 105, 110, 100, 105, 110, 103 };
         // TriggerTypeBytes = MessagePack.Serialize("aas.function.trigger");
         private static ReadOnlySpan<byte> TriggerTypeBytes => new byte[] { 180, 97, 97, 115, 46, 102, 117, 110, 99, 116, 105, 111, 110, 46, 116, 114, 105, 103, 103, 101, 114 };
+        // ExtensionVersionBytes = MessagePack.Serialize("aas.function.extension_version");
+        private static ReadOnlySpan<byte> ExtensionVersionBytes => new byte[] { 190, 97, 97, 115, 46, 102, 117, 110, 99, 116, 105, 111, 110, 46, 101, 120, 116, 101, 110, 115, 105, 111, 110, 95, 118, 101, 114, 115, 105, 111, 110 };
+        // WorkerRuntimeBytes = MessagePack.Serialize("aas.function.worker_runtime");
+        private static ReadOnlySpan<byte> WorkerRuntimeBytes => new byte[] { 187, 97, 97, 115, 46, 102, 117, 110, 99, 116, 105, 111, 110, 46, 119, 111, 114, 107, 101, 114, 95, 114, 117, 110, 116, 105, 109, 101 };
 
         public override string? GetTag(string key)
         {
@@ -37,6 +41,8 @@ partial class AzureFunctionsTags
                 "aas.function.method" => FullName,
                 "aas.function.binding" => BindingSource,
                 "aas.function.trigger" => TriggerType,
+                "aas.function.extension_version" => ExtensionVersion,
+                "aas.function.worker_runtime" => WorkerRuntime,
                 _ => base.GetTag(key),
             };
         }
@@ -57,6 +63,12 @@ public override void SetTag(string key, string? value)
                 case "aas.function.trigger": 
                     TriggerType = value;
                     break;
+                case "aas.function.extension_version": 
+                    ExtensionVersion = value;
+                    break;
+                case "aas.function.worker_runtime": 
+                    WorkerRuntime = value;
+                    break;
                 case "span.kind": 
                 case "component": 
                     Logger.Value.Warning("Attempted to set readonly tag {TagName} on {TagType}. Ignoring.", key, nameof(AzureFunctionsTags));
@@ -99,6 +111,16 @@ public override void EnumerateTags<TProcessor>(ref TProcessor processor)
                 processor.Process(new TagItem<string>("aas.function.trigger", TriggerType, TriggerTypeBytes));
             }
 
+            if (ExtensionVersion is not null)
+            {
+                processor.Process(new TagItem<string>("aas.function.extension_version", ExtensionVersion, ExtensionVersionBytes));
+            }
+
+            if (WorkerRuntime is not null)
+            {
+                processor.Process(new TagItem<string>("aas.function.worker_runtime", WorkerRuntime, WorkerRuntimeBytes));
+            }
+
             base.EnumerateTags(ref processor);
         }
 
@@ -146,6 +168,20 @@ protected override void WriteAdditionalTags(System.Text.StringBuilder sb)
                   .Append(',');
             }
 
+            if (ExtensionVersion is not null)
+            {
+                sb.Append("aas.function.extension_version (tag):")
+                  .Append(ExtensionVersion)
+                  .Append(',');
+            }
+
+            if (WorkerRuntime is not null)
+            {
+                sb.Append("aas.function.worker_runtime (tag):")
+                  .Append(WorkerRuntime)
+                  .Append(',');
+            }
+
             base.WriteAdditionalTags(sb);
         }
     }

tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/TagListGenerator/AzureFunctionsTags.g.cs

@@ -26,6 +26,10 @@ partial class AzureFunctionsTags
         private static ReadOnlySpan<byte> BindingSourceBytes => new byte[] { 180, 97, 97, 115, 46, 102, 117, 110, 99, 116, 105, 111, 110, 46, 98, 105, 110, 100, 105, 110, 103 };
         // TriggerTypeBytes = MessagePack.Serialize("aas.function.trigger");
         private static ReadOnlySpan<byte> TriggerTypeBytes => new byte[] { 180, 97, 97, 115, 46, 102, 117, 110, 99, 116, 105, 111, 110, 46, 116, 114, 105, 103, 103, 101, 114 };
+        // ExtensionVersionBytes = MessagePack.Serialize("aas.function.extension_version");
+        private static ReadOnlySpan<byte> ExtensionVersionBytes => new byte[] { 190, 97, 97, 115, 46, 102, 117, 110, 99, 116, 105, 111, 110, 46, 101, 120, 116, 101, 110, 115, 105, 111, 110, 95, 118, 101, 114, 115, 105, 111, 110 };
+        // WorkerRuntimeBytes = MessagePack.Serialize("aas.function.worker_runtime");
+        private static ReadOnlySpan<byte> WorkerRuntimeBytes => new byte[] { 187, 97, 97, 115, 46, 102, 117, 110, 99, 116, 105, 111, 110, 46, 119, 111, 114, 107, 101, 114, 95, 114, 117, 110, 116, 105, 109, 101 };
 
         public override string? GetTag(string key)
         {
@@ -37,6 +41,8 @@ partial class AzureFunctionsTags
                 "aas.function.method" => FullName,
                 "aas.function.binding" => BindingSource,
                 "aas.function.trigger" => TriggerType,
+                "aas.function.extension_version" => ExtensionVersion,
+                "aas.function.worker_runtime" => WorkerRuntime,
                 _ => base.GetTag(key),
             };
         }
@@ -57,6 +63,12 @@ public override void SetTag(string key, string? value)
                 case "aas.function.trigger": 
                     TriggerType = value;
                     break;
+                case "aas.function.extension_version": 
+                    ExtensionVersion = value;
+                    break;
+                case "aas.function.worker_runtime": 
+                    WorkerRuntime = value;
+                    break;
                 case "span.kind": 
                 case "component": 
                     Logger.Value.Warning("Attempted to set readonly tag {TagName} on {TagType}. Ignoring.", key, nameof(AzureFunctionsTags));
@@ -99,6 +111,16 @@ public override void EnumerateTags<TProcessor>(ref TProcessor processor)
                 processor.Process(new TagItem<string>("aas.function.trigger", TriggerType, TriggerTypeBytes));
             }
 
+            if (ExtensionVersion is not null)
+            {
+                processor.Process(new TagItem<string>("aas.function.extension_version", ExtensionVersion, ExtensionVersionBytes));
+            }
+
+            if (WorkerRuntime is not null)
+            {
+                processor.Process(new TagItem<string>("aas.function.worker_runtime", WorkerRuntime, WorkerRuntimeBytes));
+            }
+
             base.EnumerateTags(ref processor);
         }
 
@@ -146,6 +168,20 @@ protected override void WriteAdditionalTags(System.Text.StringBuilder sb)
                   .Append(',');
             }
 
+            if (ExtensionVersion is not null)
+            {
+                sb.Append("aas.function.extension_version (tag):")
+                  .Append(ExtensionVersion)
+                  .Append(',');
+            }
+
+            if (WorkerRuntime is not null)
+            {
+                sb.Append("aas.function.worker_runtime (tag):")
+                  .Append(WorkerRuntime)
+                  .Append(',');
+            }
+
             base.WriteAdditionalTags(sb);
         }
     }

tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/TagListGenerator/AzureFunctionsTags.g.cs

@@ -26,6 +26,10 @@ partial class AzureFunctionsTags
         private static ReadOnlySpan<byte> BindingSourceBytes => new byte[] { 180, 97, 97, 115, 46, 102, 117, 110, 99, 116, 105, 111, 110, 46, 98, 105, 110, 100, 105, 110, 103 };
         // TriggerTypeBytes = MessagePack.Serialize("aas.function.trigger");
         private static ReadOnlySpan<byte> TriggerTypeBytes => new byte[] { 180, 97, 97, 115, 46, 102, 117, 110, 99, 116, 105, 111, 110, 46, 116, 114, 105, 103, 103, 101, 114 };
+        // ExtensionVersionBytes = MessagePack.Serialize("aas.function.extension_version");
+        private static ReadOnlySpan<byte> ExtensionVersionBytes => new byte[] { 190, 97, 97, 115, 46, 102, 117, 110, 99, 116, 105, 111, 110, 46, 101, 120, 116, 101, 110, 115, 105, 111, 110, 95, 118, 101, 114, 115, 105, 111, 110 };
+        // WorkerRuntimeBytes = MessagePack.Serialize("aas.function.worker_runtime");
+        private static ReadOnlySpan<byte> WorkerRuntimeBytes => new byte[] { 187, 97, 97, 115, 46, 102, 117, 110, 99, 116, 105, 111, 110, 46, 119, 111, 114, 107, 101, 114, 95, 114, 117, 110, 116, 105, 109, 101 };
 
         public override string? GetTag(string key)
         {
@@ -37,6 +41,8 @@ partial class AzureFunctionsTags
                 "aas.function.method" => FullName,
                 "aas.function.binding" => BindingSource,
                 "aas.function.trigger" => TriggerType,
+                "aas.function.extension_version" => ExtensionVersion,
+                "aas.function.worker_runtime" => WorkerRuntime,
                 _ => base.GetTag(key),
             };
         }
@@ -57,6 +63,12 @@ public override void SetTag(string key, string? value)
                 case "aas.function.trigger": 
                     TriggerType = value;
                     break;
+                case "aas.function.extension_version": 
+                    ExtensionVersion = value;
+                    break;
+                case "aas.function.worker_runtime": 
+                    WorkerRuntime = value;
+                    break;
                 case "span.kind": 
                 case "component": 
                     Logger.Value.Warning("Attempted to set readonly tag {TagName} on {TagType}. Ignoring.", key, nameof(AzureFunctionsTags));
@@ -99,6 +111,16 @@ public override void EnumerateTags<TProcessor>(ref TProcessor processor)
                 processor.Process(new TagItem<string>("aas.function.trigger", TriggerType, TriggerTypeBytes));
             }
 
+            if (ExtensionVersion is not null)
+            {
+                processor.Process(new TagItem<string>("aas.function.extension_version", ExtensionVersion, ExtensionVersionBytes));
+            }
+
+            if (WorkerRuntime is not null)
+            {
+                processor.Process(new TagItem<string>("aas.function.worker_runtime", WorkerRuntime, WorkerRuntimeBytes));
+            }
+
             base.EnumerateTags(ref processor);
         }
 
@@ -146,6 +168,20 @@ protected override void WriteAdditionalTags(System.Text.StringBuilder sb)
                   .Append(',');
             }
 
+            if (ExtensionVersion is not null)
+            {
+                sb.Append("aas.function.extension_version (tag):")
+                  .Append(ExtensionVersion)
+                  .Append(',');
+            }
+
+            if (WorkerRuntime is not null)
+            {
+                sb.Append("aas.function.worker_runtime (tag):")
+                  .Append(WorkerRuntime)
+                  .Append(',');
+            }
+
             base.WriteAdditionalTags(sb);
         }
     }