Hotfix v3.26.3 - Update CI with Windows signing remediations (#7527)

## Summary of changes

This cherry-picks the commits related to resolving the issues that we
had with not correctly signing Windows artifacts.

## Reason for change

The remediations were only on `master` causing the hotfix to not be
signed again. This resolves that.

## Implementation details

`git cherry-pick` the three commits

## Test coverage

If the build works and says that everything gets correctly signed again
then 👍

## Other details
<!-- Fixes #{issue} -->


<!--  ⚠️ Note:

Where possible, please obtain 2 approvals prior to merging. Unless
CODEOWNERS specifies otherwise, for external teams it is typically best
to have one review from a team member, and one review from apm-dotnet.
Trivial changes do not require 2 reviews.

MergeQueue is NOT enabled in this repository. If you have write access
to the repo, the PR has 1-2 approvals (see above), and all of the
required checks have passed, you can use the Squash and Merge button to
merge the PR. If you don't have write access, or you need help, reach
out in the #apm-dotnet channel in Slack.
-->

---------

Co-authored-by: Andrew Lock <[email protected]>
Co-authored-by: Zach Montoya <[email protected]>
Co-authored-by: NachoEchevarria <[email protected]>
Co-authored-by: Lucas Pimentel <[email protected]>

Author: 5 people

.gitlab-ci.yml

@@ -36,7 +36,7 @@ build:
         -e AWS_NETWORKING=true `
         -e SIGN_WINDOWS=true `
         -e NUGET_CERT_REVOCATION_MODE=offline `
-        registry.ddbuild.io/images/mirror/datadog/dd-trace-dotnet-docker-build:dotnet10-preview7 `
+        registry.ddbuild.io/images/mirror/datadog/dd-trace-dotnet-docker-build:dotnet10-preview7.1 `
         Info Clean BuildTracerHome BuildProfilerHome BuildNativeLoader BuildDdDotnet PublishFleetInstaller PackageTracerHome ZipSymbols SignDlls SignMsi DownloadWinSsiTelemetryForwarder
     - mkdir artifacts-out
     - xcopy /e/s build-out\${CI_JOB_ID}\*.* artifacts-out

tracer/build/_build/Build.GitHub.cs

@@ -1182,7 +1182,7 @@ static async Task DownloadAzureArtifact(AbsolutePath outputDirectory, BuildArtif
 
         Console.WriteLine($"{artifact.Name} downloaded. Extracting to {outputDirectory}...");
 
-        UncompressZip(zipPath, outputDirectory);
+        UncompressZipQuiet(zipPath, outputDirectory);
 
         Console.WriteLine($"Artifact download complete");
     }

tracer/build/_build/Build.Gitlab.cs

@@ -88,6 +88,8 @@ partial class Build
 
     void SignFiles(IReadOnlyCollection<AbsolutePath> filesToSign)
     {
+        const string validSignature = "59063C826DAA5B628B5CE8A2B32015019F164BF0";
+
         Logger.Information("Signing {Count} binaries...", filesToSign.Count);
         filesToSign.ForEach(file => SignBinary(file));
         Logger.Information("Binary signing complete");
@@ -102,9 +104,48 @@ void SignBinary(AbsolutePath binaryPath)
                     logOutput: false,
                     logInvocation: false);
             signProcess.WaitForExit();
+
+            var output = signProcess.Output.Select(o => o.Text);
+            foreach (var line in output)
+            {
+                Logger.Information("[dd-wcs] {Line}", line);
+
+                // dd-wcs will return 0 even if there are errors
+                if (line.StartsWith("ERROR:", StringComparison.OrdinalIgnoreCase))
+                {
+                    throw new Exception($"Error found when signing {binaryPath}: {line}");
+                }
+            }
+
             if (signProcess.ExitCode == 0)
             {
-                PowerShellTasks.PowerShell($"Get-AuthenticodeSignature {binaryPath}");
+                var status = PowerShellTasks.PowerShell(
+                    $"(Get-AuthenticodeSignature '{binaryPath}').Status",
+                    logOutput: false,
+                    logInvocation: false);
+
+                var statusValue = status.Select(o => o.Text).FirstOrDefault(l => !string.IsNullOrEmpty(l))?.Trim();
+
+                if (!string.Equals(statusValue, "Valid", StringComparison.OrdinalIgnoreCase))
+                {
+                    throw new Exception($"Signature verification failed for {binaryPath}. Status: {statusValue ?? "Empty"}");
+                }
+
+                var print = PowerShellTasks.PowerShell(
+                    $"(Get-AuthenticodeSignature '{binaryPath}').SignerCertificate.Thumbprint",
+                    logOutput: false,
+                    logInvocation: false);
+
+                var printValue = print.Select(o => o.Text).FirstOrDefault(l => !string.IsNullOrEmpty(l))?.Trim();
+
+                if (!string.Equals(printValue, validSignature, StringComparison.OrdinalIgnoreCase))
+                {
+                    throw new Exception($"Signature verification failed for {binaryPath}. Signature: {printValue ?? "Empty"}");
+                }
+                else
+                {
+                    Logger.Information($"Signing verfication of {binaryPath} succedeed. Signature: {printValue}", binaryPath);
+                }
             }
             else
             {

tracer/build/_build/Build.Steps.cs

@@ -10,6 +10,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using CodeGenerators;
+using ICSharpCode.SharpZipLib.Zip;
 using LogParsing;
 using Mono.Cecil;
 using Nuke.Common;
@@ -275,14 +276,15 @@ bool RequiresThoroughTesting()
                 NuGetTasks.NuGetRestore(s => s
                     .SetTargetPath(Solution)
                     .SetVerbosity(NuGetVerbosity.Normal)
+                    .SetProcessLogOutput(!IsServerBuild)
                     .When(!string.IsNullOrEmpty(NugetPackageDirectory), o =>
                         o.SetPackagesDirectory(NugetPackageDirectory)));
             }
             else
             {
                 DotNetRestore(s => s
                     .SetProjectFile(Solution)
-                    .SetVerbosity(DotNetVerbosity.Normal)
+                    .SetVerbosity(DotNetVerbosity.Minimal)
                     .SetProperty("configuration", BuildConfiguration.ToString())
                     .When(!string.IsNullOrEmpty(NugetPackageDirectory), o =>
                         o.SetPackageDirectory(NugetPackageDirectory)));
@@ -545,7 +547,7 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo
         uncompressFolderTarget ??= LibDdwafDirectory(libddwafVersion);
         Console.WriteLine($"{libDdwafZip} downloaded. Extracting to {uncompressFolderTarget}...");
 
-        UncompressZip(libDdwafZip, uncompressFolderTarget);
+        UncompressZipQuiet(libDdwafZip, uncompressFolderTarget);
     }
 
     Target CopyLibDdwaf => _ => _
@@ -2776,11 +2778,36 @@ private async Task DownloadAndExtractVcpkg(AbsolutePath destinationFolder)
         EnsureExistingParentDirectory(destinationFolder);
         var parentFolder = destinationFolder.Parent;
 
-        CompressionTasks.UncompressZip(vcpkgZip, parentFolder);
+        UncompressZipQuiet(vcpkgZip, parentFolder);
 
         RenameDirectory(parentFolder / $"vcpkg-{vcpkgVersion}", destinationFolder.Name);
     }
 
+    // Quiet version of CompressionTasks:UncompressZip, which displays a log line for every created directory when uncompressing
+    private static void UncompressZipQuiet(string archiveFile, string directory)
+    {
+        Logger.Information("Uncompressing {File} to {Directory} ...", Path.GetFileName(archiveFile), directory);
+
+        using var fileStream = File.OpenRead(archiveFile);
+        using var zipFile = new ZipFile(fileStream);
+
+        var entries = zipFile.Cast<ZipEntry>().Where(x => !x.IsDirectory);
+        foreach (var entry in entries)
+        {
+            var file = PathConstruction.Combine(directory, entry.Name);
+            var path = Path.GetDirectoryName(file);
+
+            if (!Directory.Exists(path) && !string.IsNullOrEmpty(path))
+                {
+                    Directory.CreateDirectory(path);
+                }
+
+            using var entryStream = zipFile.GetInputStream(entry);
+            using var outputStream = File.Open(file, FileMode.Create);
+            entryStream.CopyTo(outputStream);
+        }
+    }
+
     public static class LibdatadogLogParser
     {
         private static readonly JsonSerializerOptions Options = new()

tracer/build/_build/docker/gitlab/gitlab.windows.dockerfile

@@ -41,8 +41,8 @@ RUN powershell -Command .\install_dotnet.ps1  -Version $ENV:DOTNET_VERSION -Sha5
 # Java and code signing tool environment variables
 ENV JAVA_VERSION "17.0.8"
 ENV JAVA_SHA256 "db6e7e7506296b8a2338f6047fdc94bf4bbc147b7a3574d9a035c3271ae1a92b"
-ENV WINSIGN_VERSION "0.2.3"
-ENV WINSIGN_SHA256 "8091cd41e8e91b8a6b2ec8c2031b12ea4ca42897b972f9f46c2be6ae4c9961f7"
+ENV WINSIGN_VERSION "0.3.5"
+ENV WINSIGN_SHA256 "b2ba5127a5c5141e04d42444ca115af4c95cc053a743caaa9b33c68dd6b13f68"
 ENV PYTHON_VERSION "3.8.2"
 
 # Install Python