From af916ccb12c0599051ed29f9e2813de12e15335d Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Tue, 2 Dec 2025 14:43:53 -0800 Subject: [PATCH 1/5] Suppress execution context flow when starting host --- src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs b/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs index f6158a6c6e..af175926d2 100644 --- a/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs +++ b/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs @@ -328,6 +328,9 @@ private async Task StartHostAsync(CancellationToken cancellationToken, int attem /// private async Task UnsynchronizedStartHostAsync(ScriptHostStartupOperation activeOperation, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal) { + // We may be started from a variety of contexts, some of which may carry AsyncLocal state (such as Activity). We do not want any of this + // to flow into the host startup logic, so we suppress the flow of the ExecutionContext. + using AsyncFlowControl flow = System.Threading.ExecutionContext.SuppressFlow(); await CheckFileSystemAsync(); if (ShutdownRequested) { From a8f413a0a281b8f10e6f247a2ec1ba02943477e9 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 3 Dec 2025 08:47:09 -0800 Subject: [PATCH 2/5] Update release_notes.md --- release_notes.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release_notes.md b/release_notes.md index 797c961128..b6e6498e36 100644 --- a/release_notes.md +++ b/release_notes.md @@ -7,4 +7,6 @@ - Add JitTrace Files for v4.1045 - Throw exception instead of timing out when worker channel exits before initializing gRPC (#10937) - Adding empty remote message check in the SystemLogger (#11473) -- Fix `webPubSubTrigger`'s for Flex consumption sku (#11489) \ No newline at end of file +- Fix `webPubSubTrigger`'s for Flex consumption sku (#11489) +- Suppress execution context flow into script host start (#11498) +- Add AzureWebJobsStorage health check (#11471) From 690670dd09d52d351c6c40eba2b70487b1c938fc Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 3 Dec 2025 09:21:42 -0800 Subject: [PATCH 3/5] Fix thread switch on AsyncFlowControl disposal --- .../WebJobsScriptHostService.cs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs b/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs index af175926d2..1ce7015ac2 100644 --- a/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs +++ b/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs @@ -326,11 +326,26 @@ private async Task StartHostAsync(CancellationToken cancellationToken, int attem /// before calling this method. Host starts and restarts must be synchronous to prevent orphaned /// hosts or an incorrect ActiveHost. /// - private async Task UnsynchronizedStartHostAsync(ScriptHostStartupOperation activeOperation, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal) + private Task UnsynchronizedStartHostAsync(ScriptHostStartupOperation activeOperation, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal) { // We may be started from a variety of contexts, some of which may carry AsyncLocal state (such as Activity). We do not want any of this // to flow into the host startup logic, so we suppress the flow of the ExecutionContext. - using AsyncFlowControl flow = System.Threading.ExecutionContext.SuppressFlow(); + Task start; + using (System.Threading.ExecutionContext.SuppressFlow()) + { + start = UnsynchronizedStartHostCoreAsync(activeOperation, attemptCount, startupMode); + } + + return start; + } + + /// + /// Starts the host without taking a lock. Callers must take a lock on _hostStartSemaphore + /// before calling this method. Host starts and restarts must be synchronous to prevent orphaned + /// hosts or an incorrect ActiveHost. + /// + private async Task UnsynchronizedStartHostCoreAsync(ScriptHostStartupOperation activeOperation, int attemptCount, JobHostStartupMode startupMode) + { await CheckFileSystemAsync(); if (ShutdownRequested) { From 9d047077f8d97ba8447a3da7867f967589ea871a Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 3 Dec 2025 11:15:51 -0800 Subject: [PATCH 4/5] Add Task.Yield --- .../WebJobsScriptHostService.cs | 1 + .../WebJobsScriptHostServiceTests.cs | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs b/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs index 1ce7015ac2..83465523cb 100644 --- a/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs +++ b/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs @@ -346,6 +346,7 @@ private Task UnsynchronizedStartHostAsync(ScriptHostStartupOperation activeOpera /// private async Task UnsynchronizedStartHostCoreAsync(ScriptHostStartupOperation activeOperation, int attemptCount, JobHostStartupMode startupMode) { + await Task.Yield(); // ensure any async context is properly suppressed. await CheckFileSystemAsync(); if (ShutdownRequested) { diff --git a/test/WebJobs.Script.Tests/WebJobsScriptHostServiceTests.cs b/test/WebJobs.Script.Tests/WebJobsScriptHostServiceTests.cs index 7c86ad54d0..6f2314deac 100644 --- a/test/WebJobs.Script.Tests/WebJobsScriptHostServiceTests.cs +++ b/test/WebJobs.Script.Tests/WebJobsScriptHostServiceTests.cs @@ -105,6 +105,15 @@ private Mock CreateMockHost(SemaphoreSlim disposedSemaphore = null) public async Task StartAsync_Succeeds() { var hostBuilder = new Mock(); + _host.Setup(h => h.StartAsync(It.IsAny())) + .Callback(() => + { + if (Activity.Current != null) + { + throw new InvalidOperationException("Activity flowed into host start."); + } + }) + .Returns(Task.CompletedTask); hostBuilder.Setup(b => b.BuildHost(It.IsAny(), It.IsAny())).Returns(_host.Object); _webHostLoggerProvider = new TestLoggerProvider(); @@ -124,6 +133,8 @@ public async Task StartAsync_Succeeds() _hostBuiltChangeTokenSource, _hostBuiltChangeTokenSourceResolverOptions); + using Activity activity = new("TestActivity_ShouldNotFlowIntoHostStart"); + activity.Start(); await _hostService.StartAsync(CancellationToken.None); // add general post startup validations here From 038b555da98ae79096ce013406fb8c3c030724a0 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 3 Dec 2025 12:49:03 -0800 Subject: [PATCH 5/5] Fix unit tests --- .../TaskExtensions.cs | 39 +++++++ .../WebJobsScriptHostServiceTests.cs | 101 ++++++++---------- 2 files changed, 81 insertions(+), 59 deletions(-) create mode 100644 test/WebJobs.Script.Tests.Shared/TaskExtensions.cs diff --git a/test/WebJobs.Script.Tests.Shared/TaskExtensions.cs b/test/WebJobs.Script.Tests.Shared/TaskExtensions.cs new file mode 100644 index 0000000000..1e7ab75caa --- /dev/null +++ b/test/WebJobs.Script.Tests.Shared/TaskExtensions.cs @@ -0,0 +1,39 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.Threading.Tasks; + +namespace Microsoft.WebJobs.Script.Tests +{ + public static class TaskExtensions + { + /// + /// Waits for a task to complete with a test-appropriate timeout. If a debugger is attached, waits indefinitely. + /// + /// The task to wait on. + /// A representing the asynchronous wait. + public static Task TestWaitAsync(this Task task) + { + return task.WaitAsync(TimeSpan.FromSeconds(10)); + } + + /// + /// Waits for a task to complete with a test-appropriate timeout. If a debugger is attached, waits indefinitely. + /// + /// The task to wait on. + /// The timeout to use if no debugger is attached. + /// A representing the asynchronous wait. + public static Task TestWaitAsync(this Task task, TimeSpan timeout) + { + ArgumentNullException.ThrowIfNull(task); + if (Debugger.IsAttached) + { + return task; + } + + return task.WaitAsync(timeout); + } + } +} diff --git a/test/WebJobs.Script.Tests/WebJobsScriptHostServiceTests.cs b/test/WebJobs.Script.Tests/WebJobsScriptHostServiceTests.cs index 6f2314deac..bc0548c1dc 100644 --- a/test/WebJobs.Script.Tests/WebJobsScriptHostServiceTests.cs +++ b/test/WebJobs.Script.Tests/WebJobsScriptHostServiceTests.cs @@ -201,12 +201,9 @@ public async Task HostRestart_Specialization_Succeeds() await _hostService.StartAsync(CancellationToken.None); Assert.True(AreRequiredMetricsGenerated(metricsLogger)); - Thread restartHostThread = new Thread(new ThreadStart(RestartHost)); - Thread specializeHostThread = new Thread(new ThreadStart(SpecializeHost)); - restartHostThread.Start(); - specializeHostThread.Start(); - restartHostThread.Join(); - specializeHostThread.Join(); + Task restartHost = _hostService.RestartHostAsync("test", default); + Task specializeHost = SpecializeHostAsync(); + await Task.WhenAll(restartHost, specializeHost).TestWaitAsync(TimeSpan.FromSeconds(60)); // this is a slow test for some reason. var logMessages = _webHostLoggerProvider.GetAllLogMessages().Where(m => m.FormattedMessage.Contains("Restarting host.")); Assert.Equal(2, logMessages.Count()); @@ -215,9 +212,8 @@ public async Task HostRestart_Specialization_Succeeds() [Fact] public async Task HostRestart_DuringInitializationWithError_Recovers() { - SemaphoreSlim semaphore = new SemaphoreSlim(1, 1); - await semaphore.WaitAsync(); - bool waitingInHostBuilder = false; + TaskCompletionSource startBlock = new(); + TaskCompletionSource initialStart = new(); var metricsLogger = new TestMetricsLogger(); // Have the first host start, but pause. We'll issue a restart, then let @@ -227,8 +223,8 @@ public async Task HostRestart_DuringInitializationWithError_Recovers() .Setup(h => h.StartAsync(It.IsAny())) .Returns(async () => { - waitingInHostBuilder = true; - await semaphore.WaitAsync(); + initialStart.SetResult(); + await startBlock.Task; throw new InvalidOperationException("Something happened at startup!"); }); @@ -237,17 +233,14 @@ public async Task HostRestart_DuringInitializationWithError_Recovers() .Setup(h => h.StartAsync(It.IsAny())) .Returns(() => Task.CompletedTask); - var hostBuilder = new Mock(); - hostBuilder.SetupSequence(b => b.BuildHost(It.IsAny(), It.IsAny())) - .Returns(hostA.Object) - .Returns(hostB.Object); + OrderedScriptHostBuilder hostBuilder = new(hostA.Object, hostB.Object); _webHostLoggerProvider = new TestLoggerProvider(); _loggerFactory = new LoggerFactory(); _loggerFactory.AddProvider(_webHostLoggerProvider); _hostService = new WebJobsScriptHostService( - _monitor, hostBuilder.Object, _loggerFactory, + _monitor, hostBuilder, _loggerFactory, _mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions, metricsLogger, new Mock().Object, @@ -256,20 +249,17 @@ public async Task HostRestart_DuringInitializationWithError_Recovers() TestLoggerProvider hostALogger = hostA.Object.GetTestLoggerProvider(); TestLoggerProvider hostBLogger = hostB.Object.GetTestLoggerProvider(); - Task initialStart = _hostService.StartAsync(CancellationToken.None); - - Thread restartHostThread = new Thread(new ThreadStart(RestartHost)); - restartHostThread.Start(); - await TestHelpers.Await(() => waitingInHostBuilder); + Task firstStart = _hostService.StartAsync(CancellationToken.None); + await initialStart.Task.TestWaitAsync(); + Task restartTask = _hostService.RestartHostAsync("test", default); // Now let the first host continue. It will throw, which should be correctly handled. - semaphore.Release(); + startBlock.SetResult(); - await TestHelpers.Await(() => _hostService.State == ScriptHostState.Running); - - restartHostThread.Join(); + await restartTask; + Assert.Equal(ScriptHostState.Running, _hostService.State); - await initialStart; + await firstStart; Assert.True(AreRequiredMetricsGenerated(metricsLogger)); // Make sure the error was logged to the correct logger @@ -291,21 +281,19 @@ public async Task HostRestart_DuringInitializationWithError_Recovers() [Fact] public async Task HostRestart_DuringInitialization_Cancels() { - SemaphoreSlim semaphore = new SemaphoreSlim(1, 1); - await semaphore.WaitAsync(); - bool waitingInHostBuilder = false; + TaskCompletionSource startBlock = new(); + TaskCompletionSource initialStart = new(); var metricsLogger = new TestMetricsLogger(); // Have the first host start, but pause. We'll issue a restart, then - // let this first host continue and see the cancelation token has fired. + // let this first host continue and see the cancellation token has fired. var hostA = CreateMockHost(); hostA .Setup(h => h.StartAsync(It.IsAny())) .Returns(async ct => { - waitingInHostBuilder = true; - await semaphore.WaitAsync(); - ct.ThrowIfCancellationRequested(); + initialStart.SetResult(); + await startBlock.Task.WaitAsync(ct); }); var hostB = CreateMockHost(); @@ -313,39 +301,28 @@ public async Task HostRestart_DuringInitialization_Cancels() .Setup(h => h.StartAsync(It.IsAny())) .Returns(() => Task.CompletedTask); - var hostBuilder = new Mock(); - hostBuilder.SetupSequence(b => b.BuildHost(It.IsAny(), It.IsAny())) - .Returns(hostA.Object) - .Returns(hostB.Object); + OrderedScriptHostBuilder hostBuilder = new(hostA.Object, hostB.Object); _webHostLoggerProvider = new TestLoggerProvider(); _loggerFactory = new LoggerFactory(); _loggerFactory.AddProvider(_webHostLoggerProvider); _hostService = new WebJobsScriptHostService( - _monitor, hostBuilder.Object, _loggerFactory, + _monitor, hostBuilder, _loggerFactory, _mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions, metricsLogger, new Mock().Object, _mockConfig, new TestScriptEventManager(), _hostMetrics, _functionsHostingConfigOptions, _hostBuiltChangeTokenSource, _hostBuiltChangeTokenSourceResolverOptions); TestLoggerProvider hostALogger = hostA.Object.GetTestLoggerProvider(); + Task firstStart = _hostService.StartAsync(default); + await initialStart.Task.TestWaitAsync(); + Task restartTask = _hostService.RestartHostAsync("test", default); - Task initialStart = _hostService.StartAsync(CancellationToken.None); - - Thread restartHostThread = new Thread(new ThreadStart(RestartHost)); - restartHostThread.Start(); - await TestHelpers.Await(() => waitingInHostBuilder); - await TestHelpers.Await(() => _webHostLoggerProvider.GetLog().Contains("Restart requested.")); - - // Now let the first host continue. It will see that startup is canceled. - semaphore.Release(); - - await TestHelpers.Await(() => _hostService.State == ScriptHostState.Running); - - restartHostThread.Join(); + await restartTask; + Assert.Equal(ScriptHostState.Running, _hostService.State); - await initialStart; + await firstStart; Assert.True(AreRequiredMetricsGenerated(metricsLogger)); // Make sure the error was logged to the correct logger @@ -593,12 +570,7 @@ public async Task DependencyTrackingTelemetryModule_Race() await listenerTask; } - public void RestartHost() - { - _hostService.RestartHostAsync("test", CancellationToken.None).Wait(); - } - - public void SpecializeHost() + public Task SpecializeHostAsync() { var mockScriptWebHostEnvironment = new Mock(); Mock mockConfiguration = new Mock(); @@ -611,7 +583,7 @@ public void SpecializeHost() var hostNameProvider = new HostNameProvider(mockEnvironment.Object); var mockApplicationLifetime = new Mock(MockBehavior.Strict); var manager = new StandbyManager(_hostService, mockLanguageWorkerChannelManager.Object, mockConfiguration.Object, mockScriptWebHostEnvironment.Object, mockEnvironment.Object, _monitor, testLogger, hostNameProvider, mockApplicationLifetime.Object, new TestMetricsLogger()); - manager.SpecializeHostAsync().Wait(); + return manager.SpecializeHostAsync(); } private bool AreRequiredMetricsGenerated(TestMetricsLogger testMetricsLogger) @@ -657,5 +629,16 @@ public IHost BuildHost(bool skipHostStartup, bool skipHostConfigurationParsing) return null; } } + + // Moq SetupSequence doesn't work with execution context suppression. + private class OrderedScriptHostBuilder(params IHost[] hosts) : IScriptHostBuilder + { + private readonly Queue _hosts = new(hosts); + + public IHost BuildHost(bool skipHostStartup, bool skipHostConfigurationParsing) + { + return _hosts.Dequeue(); + } + } } } \ No newline at end of file