diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/AddRepoOperation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/AddRepoOperation.cs new file mode 100644 index 0000000000..e85b5ad9e2 --- /dev/null +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/AddRepoOperation.cs @@ -0,0 +1,109 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Maestro.Common; +using Microsoft.DotNet.Darc.Helpers; +using Microsoft.DotNet.Darc.Options.VirtualMonoRepo; +using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; +using Microsoft.Extensions.Logging; + +#nullable enable +namespace Microsoft.DotNet.Darc.Operations.VirtualMonoRepo; + +internal class AddRepoOperation : Operation +{ + private readonly AddRepoCommandLineOptions _options; + private readonly IVmrInitializer _vmrInitializer; + private readonly IVmrInfo _vmrInfo; + private readonly ILogger _logger; + + public AddRepoOperation( + AddRepoCommandLineOptions options, + IVmrInitializer vmrInitializer, + IVmrInfo vmrInfo, + ILogger logger) + { + _options = options; + _vmrInitializer = vmrInitializer; + _vmrInfo = vmrInfo; + _logger = logger; + } + + public override async Task ExecuteAsync() + { + var repositories = _options.Repositories.ToList(); + + if (!repositories.Any()) + { + _logger.LogError("Please specify at least one repository to add"); + return Constants.ErrorCode; + } + + // Repository names are in the form of URI:REVISION where URI is the git repository URL + // and REVISION is a git ref (commit SHA, branch, or tag) + foreach (var repository in repositories) + { + var parts = repository.Split(':', 2); + if (parts.Length != 2) + { + _logger.LogError("Repository '{repository}' must be in the format URI:REVISION", repository); + return Constants.ErrorCode; + } + + string uri = parts[0]; + string revision = parts[1]; + + // For URIs starting with https://, we need to reconstruct the full URI + // since the split on ':' would have separated it + if (uri == "https" || uri == "http") + { + // The original input had a URI with protocol, find the last : to split properly + int lastColonIndex = repository.LastIndexOf(':'); + if (lastColonIndex <= uri.Length + 2) // +2 for "://" + { + _logger.LogError("Repository '{repository}' must be in the format URI:REVISION", repository); + return Constants.ErrorCode; + } + + uri = repository.Substring(0, lastColonIndex); + revision = repository.Substring(lastColonIndex + 1); + } + + try + { + // Extract repo name from URI + var (repoName, _) = GitRepoUrlUtils.GetRepoNameAndOwner(uri); + + var sourceMappingsPath = _vmrInfo.VmrPath / VmrInfo.DefaultRelativeSourceMappingsPath; + + await _vmrInitializer.InitializeRepository( + repoName, + revision, + uri, + sourceMappingsPath, + new CodeFlowParameters( + Array.Empty(), + VmrInfo.ThirdPartyNoticesFileName, + GenerateCodeOwners: false, + GenerateCredScanSuppressions: true), + CancellationToken.None); + + _logger.LogInformation("Successfully added repository '{repoName}' from '{uri}' at revision '{revision}'", repoName, uri, revision); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to add repository from '{uri}'", uri); + return Constants.ErrorCode; + } + } + + return Constants.SuccessCode; + } +} diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/RemoveRepoOperation.cs similarity index 57% rename from src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs rename to src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/RemoveRepoOperation.cs index bf5185e18e..1003c3519d 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/RemoveRepoOperation.cs @@ -5,7 +5,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.DotNet.Darc.Options.VirtualMonoRepo; -using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; using Microsoft.DotNet.DarcLib.VirtualMonoRepo; using Microsoft.Extensions.Logging; @@ -13,19 +12,19 @@ #nullable enable namespace Microsoft.DotNet.Darc.Operations.VirtualMonoRepo; -internal class InitializeOperation : VmrOperationBase +internal class RemoveRepoOperation : VmrOperationBase { - private readonly InitializeCommandLineOptions _options; - private readonly IVmrInitializer _vmrInitializer; + private readonly RemoveRepoCommandLineOptions _options; + private readonly IVmrRemover _vmrRemover; - public InitializeOperation( - InitializeCommandLineOptions options, - IVmrInitializer vmrInitializer, - ILogger logger) + public RemoveRepoOperation( + RemoveRepoCommandLineOptions options, + IVmrRemover vmrRemover, + ILogger logger) : base(options, logger) { _options = options; - _vmrInitializer = vmrInitializer; + _vmrRemover = vmrRemover; } protected override async Task ExecuteInternalAsync( @@ -34,15 +33,13 @@ protected override async Task ExecuteInternalAsync( IReadOnlyCollection additionalRemotes, CancellationToken cancellationToken) { - await _vmrInitializer.InitializeRepository( + await _vmrRemover.RemoveRepository( repoName, - targetRevision, - new NativePath(_options.SourceMappings), new CodeFlowParameters( additionalRemotes, - _options.TpnTemplate, - _options.GenerateCodeowners, - _options.GenerateCredScanSuppressions), + VmrInfo.ThirdPartyNoticesFileName, + GenerateCodeOwners: false, + GenerateCredScanSuppressions: true), cancellationToken); } } diff --git a/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/AddRepoCommandLineOptions.cs b/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/AddRepoCommandLineOptions.cs new file mode 100644 index 0000000000..7ad17bd37c --- /dev/null +++ b/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/AddRepoCommandLineOptions.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using CommandLine; +using Microsoft.DotNet.Darc.Operations.VirtualMonoRepo; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.DotNet.Darc.Options.VirtualMonoRepo; + +[Verb("add-repo", HelpText = "Adds new repo(s) to the VMR that haven't been synchronized yet.")] +internal class AddRepoCommandLineOptions : VmrCommandLineOptions, IBaseVmrCommandLineOptions +{ + [Value(0, Required = true, HelpText = + "Repository URIs in the form of URI:REVISION where URI is the git repository URL (e.g., https://github.com/dotnet/runtime) and REVISION is a commit SHA or other git reference (branch, tag).")] + public IEnumerable Repositories { get; set; } + + // Required by IBaseVmrCommandLineOptions but not used for this command + public IEnumerable AdditionalRemotes { get; set; } = []; + + public override IServiceCollection RegisterServices(IServiceCollection services) + { + if (!Verbose && !Debug) + { + // Force verbose output for these commands + Verbose = true; + } + + return base.RegisterServices(services); + } +} diff --git a/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/InitializeCommandLineOptions.cs b/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/InitializeCommandLineOptions.cs deleted file mode 100644 index 3bea0825df..0000000000 --- a/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/InitializeCommandLineOptions.cs +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Generic; -using CommandLine; -using Microsoft.DotNet.Darc.Operations.VirtualMonoRepo; -using Microsoft.DotNet.DarcLib.VirtualMonoRepo; - -namespace Microsoft.DotNet.Darc.Options.VirtualMonoRepo; - -[Verb("initialize", HelpText = "Initializes new repo(s) that haven't been synchronized into the VMR yet.")] -internal class InitializeCommandLineOptions : VmrCommandLineOptions, IBaseVmrCommandLineOptions -{ - [Option("source-mappings", Required = true, HelpText = $"A path to the {VmrInfo.SourceMappingsFileName} file to be used for syncing.")] - public string SourceMappings { get; set; } - - [Option("additional-remotes", Required = false, HelpText = - "List of additional remote URIs to add to mappings in the format [mapping name]:[remote URI]. " + - "Example: installer:https://github.com/myfork/installer sdk:/local/path/to/sdk")] - [RedactFromLogging] - public IEnumerable AdditionalRemotes { get; set; } - - [Value(0, Required = true, HelpText = - "Repository names in the form of NAME or NAME:REVISION where REVISION is a commit SHA or other git reference (branch, tag). " + - "Omitting REVISION will synchronize the repo to current HEAD.")] - public IEnumerable Repositories { get; set; } - - [Option("tpn-template", Required = false, HelpText = "Path to a template for generating VMRs THIRD-PARTY-NOTICES file. Leave empty to skip generation.")] - public string TpnTemplate { get; set; } - - [Option("generate-codeowners", Required = false, HelpText = "Generate a common CODEOWNERS file for all repositories.")] - public bool GenerateCodeowners { get; set; } = false; - - [Option("generate-credscansuppressions", Required = false, HelpText = "Generate a common .config/CredScanSuppressions.json file for all repositories.")] - public bool GenerateCredScanSuppressions { get; set; } = false; -} diff --git a/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/RemoveRepoCommandLineOptions.cs b/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/RemoveRepoCommandLineOptions.cs new file mode 100644 index 0000000000..e7e5e90402 --- /dev/null +++ b/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/RemoveRepoCommandLineOptions.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using CommandLine; +using Microsoft.DotNet.Darc.Operations.VirtualMonoRepo; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.DotNet.Darc.Options.VirtualMonoRepo; + +[Verb("remove-repo", HelpText = "Removes repo(s) from the VMR.")] +internal class RemoveRepoCommandLineOptions : VmrCommandLineOptions, IBaseVmrCommandLineOptions +{ + [Value(0, Required = true, HelpText = "Repository names to remove from the VMR.")] + public IEnumerable Repositories { get; set; } + + // Required by IBaseVmrCommandLineOptions but not used for this command + public IEnumerable AdditionalRemotes { get; set; } = []; + + public override IServiceCollection RegisterServices(IServiceCollection services) + { + if (!Verbose && !Debug) + { + // Force verbose output for these commands + Verbose = true; + } + + return base.RegisterServices(services); + } +} diff --git a/src/Microsoft.DotNet.Darc/Darc/Program.cs b/src/Microsoft.DotNet.Darc/Darc/Program.cs index 43ae5775be..29b1764594 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Program.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Program.cs @@ -145,7 +145,8 @@ public static Type[] GetOptions() => // These are under the "vmr" subcommand public static Type[] GetVmrOptions() => [ - typeof(InitializeCommandLineOptions), + typeof(AddRepoCommandLineOptions), + typeof(RemoveRepoCommandLineOptions), typeof(BackflowCommandLineOptions), typeof(ForwardFlowCommandLineOptions), typeof(CherryPickCommandLineOptions), diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Models/VirtualMonoRepo/SourceMappingFile.cs b/src/Microsoft.DotNet.Darc/DarcLib/Models/VirtualMonoRepo/SourceMappingFile.cs index c7dc7d4403..ce2bd2a718 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Models/VirtualMonoRepo/SourceMappingFile.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Models/VirtualMonoRepo/SourceMappingFile.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; +using System.Text.Json.Serialization; #nullable enable namespace Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; @@ -44,7 +45,9 @@ public class SourceMappingSetting public string? DefaultRef { get; set; } public string[]? Include { get; set; } public string[]? Exclude { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public bool IgnoreDefaults { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public bool DisableSynchronization { get; set; } } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrInitializer.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrInitializer.cs index ee451a829e..561a365434 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrInitializer.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrInitializer.cs @@ -16,11 +16,13 @@ public interface IVmrInitializer /// /// Name of a repository mapping /// Revision (commit SHA, branch, tag..) onto which to synchronize, leave empty for HEAD + /// Remote URI of the repository (optional, will default to https://github.com/dotnet/{mappingName}) /// Path to the source-mappings.json file /// Record containing parameters for VMR initialization Task InitializeRepository( string mappingName, string? targetRevision, + string? remoteUri, LocalPath sourceMappingsPath, CodeFlowParameters codeFlowParameters, CancellationToken cancellationToken); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrRemover.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrRemover.cs new file mode 100644 index 0000000000..19695ef25f --- /dev/null +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrRemover.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Threading; +using System.Threading.Tasks; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; + +#nullable enable +namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo; + +public interface IVmrRemover +{ + /// + /// Removes a repository from the VMR. + /// + /// Name of a repository mapping + /// Record containing parameters for VMR operations + /// Cancellation token + Task RemoveRepository( + string mappingName, + CodeFlowParameters codeFlowParameters, + CancellationToken cancellationToken); +} diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInitializer.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInitializer.cs index 1d3a1d1a48..26262fa67b 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInitializer.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInitializer.cs @@ -2,17 +2,14 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.Collections.Generic; using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; -using Maestro.Common; using Microsoft.DotNet.DarcLib.Helpers; -using Microsoft.DotNet.DarcLib.Models; -using Microsoft.DotNet.DarcLib.Models.Darc; using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; using Microsoft.Extensions.Logging; -using Microsoft.VisualBasic; #nullable enable namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo; @@ -34,60 +31,49 @@ public class VmrInitializer : VmrManagerBase, IVmrInitializer {{Constants.AUTOMATION_COMMIT_TAG}} """; - // Message used when finalizing the initialization with a merge commit - private const string MergeCommitMessage = - $$""" - Recursive initialization for {name} / {newShaShort} - - {{Constants.AUTOMATION_COMMIT_TAG}} - """; - private readonly IVmrInfo _vmrInfo; private readonly IVmrDependencyTracker _dependencyTracker; - private readonly IVersionDetailsParser _versionDetailsParser; private readonly IRepositoryCloneManager _cloneManager; - private readonly IDependencyFileManager _dependencyFileManager; private readonly IWorkBranchFactory _workBranchFactory; private readonly IFileSystem _fileSystem; + private readonly ILocalGitClient _localGitClient; private readonly ILogger _logger; - private readonly ISourceManifest _sourceManifest; public VmrInitializer( - IVmrDependencyTracker dependencyTracker, - IVmrPatchHandler patchHandler, - IVersionDetailsParser versionDetailsParser, - IRepositoryCloneManager cloneManager, - IThirdPartyNoticesGenerator thirdPartyNoticesGenerator, - ICodeownersGenerator codeownersGenerator, - ICredScanSuppressionsGenerator credScanSuppressionsGenerator, - ILocalGitClient localGitClient, - ILocalGitRepoFactory localGitRepoFactory, - IDependencyFileManager dependencyFileManager, - IWorkBranchFactory workBranchFactory, - IFileSystem fileSystem, - ILogger logger, - ISourceManifest sourceManifest, - IVmrInfo vmrInfo) + IVmrDependencyTracker dependencyTracker, + IVmrPatchHandler patchHandler, + IRepositoryCloneManager cloneManager, + IThirdPartyNoticesGenerator thirdPartyNoticesGenerator, + ICodeownersGenerator codeownersGenerator, + ICredScanSuppressionsGenerator credScanSuppressionsGenerator, + ILocalGitClient localGitClient, + ILocalGitRepoFactory localGitRepoFactory, + IWorkBranchFactory workBranchFactory, + IFileSystem fileSystem, + ILogger logger, + IVmrInfo vmrInfo) : base(vmrInfo, dependencyTracker, patchHandler, thirdPartyNoticesGenerator, codeownersGenerator, credScanSuppressionsGenerator, localGitClient, localGitRepoFactory, logger) { _vmrInfo = vmrInfo; _dependencyTracker = dependencyTracker; - _versionDetailsParser = versionDetailsParser; _cloneManager = cloneManager; - _dependencyFileManager = dependencyFileManager; _workBranchFactory = workBranchFactory; _fileSystem = fileSystem; + _localGitClient = localGitClient; _logger = logger; - _sourceManifest = sourceManifest; } public async Task InitializeRepository( string mappingName, string? targetRevision, + string? remoteUri, LocalPath sourceMappingsPath, CodeFlowParameters codeFlowParameters, CancellationToken cancellationToken) { + // Ensure source mapping exists before initializing + await EnsureSourceMappingExistsAsync(mappingName, remoteUri, sourceMappingsPath, cancellationToken); + await _dependencyTracker.RefreshMetadataAsync(sourceMappingsPath); var mapping = _dependencyTracker.GetMapping(mappingName); @@ -124,10 +110,7 @@ public async Task InitializeRepository( "Please investigate!"); } - await InitializeRepository( - update, - codeFlowParameters, - cancellationToken); + await InitializeRepository(update, codeFlowParameters, cancellationToken); } catch (Exception) { @@ -138,14 +121,9 @@ await InitializeRepository( throw; } - string newSha = _dependencyTracker.GetDependencyVersion(mapping)?.Sha - ?? throw new Exception($"Repository {mapping.Name} was supposed to be but has not been initialized! " + - "Please make sure the sources folder is empty!"); - - var commitMessage = PrepareCommitMessage(MergeCommitMessage, mapping.Name, mapping.DefaultRemote, oldSha: null, newSha); - await workBranch.MergeBackAsync(commitMessage); + await workBranch.RebaseAsync(cancellationToken); - _logger.LogInformation("Recursive initialization for {repo} / {sha} finished", mapping.Name, newSha); + _logger.LogInformation("Repository {repo} added (staged)", mapping.Name); } private async Task InitializeRepository( @@ -176,7 +154,11 @@ private async Task InitializeRepository( TargetRevision = await clone.GetShaForRefAsync(update.TargetRevision) }; - string commitMessage = PrepareCommitMessage(InitializationCommitMessage, update.Mapping.Name, update.RemoteUri, newSha: update.TargetRevision); + string commitMessage = PrepareCommitMessage( + InitializationCommitMessage, + update.Mapping.Name, + update.RemoteUri, + newSha: update.TargetRevision); await UpdateRepoToRevisionAsync( update, @@ -191,114 +173,60 @@ await UpdateRepoToRevisionAsync( _logger.LogInformation("Initialization of {name} finished", update.Mapping.Name); } - /// - /// Recursively parses Version.Details.xml files of all repositories and returns the list of source build dependencies. - /// - private async Task> GetAllDependenciesAsync( - VmrDependencyUpdate root, - IReadOnlyCollection additionalRemotes, + private async Task EnsureSourceMappingExistsAsync( + string repoName, + string? defaultRemote, + LocalPath sourceMappingsPath, CancellationToken cancellationToken) { - var transitiveDependencies = new Dictionary + // Refresh metadata to load existing mappings + await _dependencyTracker.RefreshMetadataAsync(sourceMappingsPath); + + // Check if mapping already exists + if (_dependencyTracker.TryGetMapping(repoName, out _)) + { + _logger.LogInformation("Source mapping for '{repoName}' already exists", repoName); + return false; + } + + // Read the existing source-mappings.json file + var json = await _fileSystem.ReadAllTextAsync(sourceMappingsPath); + + var options = new JsonSerializerOptions { - { root.Mapping, root }, + AllowTrailingCommas = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + ReadCommentHandling = JsonCommentHandling.Skip, + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault | JsonIgnoreCondition.WhenWritingNull, }; - var reposToScan = new Queue(); - reposToScan.Enqueue(transitiveDependencies.Values.Single()); + var sourceMappingFile = JsonSerializer.Deserialize(json, options) + ?? throw new Exception($"Failed to deserialize {VmrInfo.SourceMappingsFileName}"); - _logger.LogInformation("Finding transitive dependencies for {mapping}:{revision}..", root.Mapping.Name, root.TargetRevision); + // Determine the default remote URL + // If not provided, use GitHub dotnet org + defaultRemote ??= $"https://github.com/dotnet/{repoName}"; - while (reposToScan.TryDequeue(out var repo)) + // Add the new mapping + var newMapping = new SourceMappingSetting { - cancellationToken.ThrowIfCancellationRequested(); - - var remotes = additionalRemotes - .Where(r => r.Mapping == repo.Mapping.Name) - .Select(r => r.RemoteUri) - .Append(repo.RemoteUri) - .Prepend(repo.Mapping.DefaultRemote) - .Distinct() - .OrderRemotesByLocalPublicOther(); - - IEnumerable? repoDependencies = null; - foreach (var remoteUri in remotes) - { - try - { - repoDependencies = (await GetRepoDependenciesAsync(remoteUri, repo.TargetRevision)) - .Where(dep => dep.SourceBuild is not null); - break; - } - catch - { - _logger.LogDebug("Could not find {file} for {mapping}:{revision} in {remote}", - VersionFiles.VersionDetailsXml, - repo.Mapping.Name, - repo.TargetRevision, - remoteUri); - } - } + Name = repoName, + DefaultRemote = defaultRemote, + }; - if (repoDependencies is null) - { - _logger.LogInformation( - "Repository {repository} does not have {file} file, skipping dependency detection.", - repo.Mapping.Name, - VersionFiles.VersionDetailsXml); - continue; - } + sourceMappingFile.Mappings.Add(newMapping); - foreach (var dependency in repoDependencies) - { - if (!_dependencyTracker.TryGetMapping(dependency.SourceBuild.RepoName, out var mapping)) - { - throw new InvalidOperationException( - $"No source mapping named '{dependency.SourceBuild.RepoName}' found " + - $"for a {VersionFiles.VersionDetailsXml} dependency of {dependency.Name}"); - } - - var update = new VmrDependencyUpdate( - mapping, - dependency.RepoUri, - dependency.Commit, - repo.Mapping, - OfficialBuildId: null, - BarId: null); - - if (transitiveDependencies.TryAdd(mapping, update)) - { - _logger.LogDebug("Detected {parent}'s dependency {name} ({uri} / {sha})", - repo.Mapping.Name, - update.Mapping.Name, - update.RemoteUri, - update.TargetRevision); - - reposToScan.Enqueue(update); - } - } - } - - _logger.LogInformation("Found {count} transitive dependencies for {mapping}:{revision}..", - transitiveDependencies.Count, - root.Mapping.Name, - root.TargetRevision); + // Write the updated source-mappings.json file + var updatedJson = JsonSerializer.Serialize(sourceMappingFile, options); + _fileSystem.WriteToFile(sourceMappingsPath, updatedJson); - return transitiveDependencies.Values; - } + // Stage the source-mappings.json file + await _localGitClient.StageAsync(_vmrInfo.VmrPath, [sourceMappingsPath], cancellationToken); - private async Task> GetRepoDependenciesAsync(string remoteRepoUri, string commitSha) - { - // Check if we have the file locally - var localVersion = _sourceManifest.Repositories.FirstOrDefault(repo => repo.RemoteUri == remoteRepoUri); - if (localVersion?.CommitSha == commitSha) - { - var path = _vmrInfo.VmrPath / VmrInfo.SourcesDir / localVersion.Path / VersionFiles.VersionDetailsXml; - var content = await _fileSystem.ReadAllTextAsync(path); - return _versionDetailsParser.ParseVersionDetailsXml(content, includePinned: true).Dependencies; - } + _logger.LogInformation("Added source mapping for '{repoName}' with remote '{defaultRemote}' and staged {file}", + repoName, defaultRemote, VmrInfo.SourceMappingsFileName); - VersionDetails versionDetails = await _dependencyFileManager.ParseVersionDetailsXmlAsync(remoteRepoUri, commitSha, includePinned: true); - return versionDetails.Dependencies; + return true; } } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs index 51350691e3..b17ea568a8 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs @@ -111,6 +111,7 @@ private static IServiceCollection AddVmrManagers( services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); + services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRemover.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRemover.cs new file mode 100644 index 0000000000..814bbed3ca --- /dev/null +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRemover.cs @@ -0,0 +1,195 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; +using Microsoft.Extensions.Logging; + +#nullable enable +namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo; + +/// +/// This class is able to remove a repository from the VMR. +/// It removes the sources, updates the source manifest, and optionally regenerates +/// third-party notices, codeowners, and credential scan suppressions. +/// +public class VmrRemover : VmrManagerBase, IVmrRemover +{ + private const string RemovalCommitMessage = + $$""" + [{0}] Removal of the repository from VMR + + {{Constants.AUTOMATION_COMMIT_TAG}} + """; + + private readonly IVmrInfo _vmrInfo; + private readonly IVmrDependencyTracker _dependencyTracker; + private readonly IWorkBranchFactory _workBranchFactory; + private readonly IFileSystem _fileSystem; + private readonly ILogger _logger; + private readonly ISourceManifest _sourceManifest; + private readonly ILocalGitClient _localGitClient; + private readonly IThirdPartyNoticesGenerator _thirdPartyNoticesGenerator; + private readonly ICodeownersGenerator _codeownersGenerator; + private readonly ICredScanSuppressionsGenerator _credScanSuppressionsGenerator; + + public VmrRemover( + IVmrDependencyTracker dependencyTracker, + IVmrPatchHandler patchHandler, + IThirdPartyNoticesGenerator thirdPartyNoticesGenerator, + ICodeownersGenerator codeownersGenerator, + ICredScanSuppressionsGenerator credScanSuppressionsGenerator, + ILocalGitClient localGitClient, + ILocalGitRepoFactory localGitRepoFactory, + IWorkBranchFactory workBranchFactory, + IFileSystem fileSystem, + ILogger logger, + ILogger baseLogger, + ISourceManifest sourceManifest, + IVmrInfo vmrInfo) + : base(vmrInfo, dependencyTracker, patchHandler, thirdPartyNoticesGenerator, codeownersGenerator, credScanSuppressionsGenerator, localGitClient, localGitRepoFactory, baseLogger) + { + _vmrInfo = vmrInfo; + _dependencyTracker = dependencyTracker; + _workBranchFactory = workBranchFactory; + _fileSystem = fileSystem; + _logger = logger; + _sourceManifest = sourceManifest; + _localGitClient = localGitClient; + _thirdPartyNoticesGenerator = thirdPartyNoticesGenerator; + _codeownersGenerator = codeownersGenerator; + _credScanSuppressionsGenerator = credScanSuppressionsGenerator; + } + + public async Task RemoveRepository( + string mappingName, + CodeFlowParameters codeFlowParameters, + CancellationToken cancellationToken) + { + await _dependencyTracker.RefreshMetadataAsync(); + var mapping = _dependencyTracker.GetMapping(mappingName); + + if (_dependencyTracker.GetDependencyVersion(mapping) is null) + { + throw new Exception($"Repository {mapping.Name} does not exist in the VMR"); + } + + var workBranchName = $"remove/{mapping.Name}"; + IWorkBranch workBranch = await _workBranchFactory.CreateWorkBranchAsync(GetLocalVmr(), workBranchName); + + try + { + _logger.LogInformation("Removing {name} from the VMR..", mapping.Name); + + var sourcesPath = _vmrInfo.GetRepoSourcesPath(mapping); + if (_fileSystem.DirectoryExists(sourcesPath)) + { + _logger.LogInformation("Removing source directory {path}", sourcesPath); + _fileSystem.DeleteDirectory(sourcesPath, recursive: true); + } + else + { + _logger.LogWarning("Source directory {path} does not exist", sourcesPath); + } + + // Remove from source manifest + _sourceManifest.RemoveRepository(mapping.Name); + _fileSystem.WriteToFile(_vmrInfo.SourceManifestPath, _sourceManifest.ToJson()); + + var pathsToStage = new List + { + _vmrInfo.SourceManifestPath, + sourcesPath + }; + + // Remove source mapping + var sourceMappingsPath = _vmrInfo.VmrPath / VmrInfo.DefaultRelativeSourceMappingsPath; + if (await RemoveSourceMappingAsync(mapping.Name, sourceMappingsPath, cancellationToken)) + { + pathsToStage.Add(sourceMappingsPath); + } + + await _localGitClient.StageAsync(_vmrInfo.VmrPath, pathsToStage, cancellationToken); + + cancellationToken.ThrowIfCancellationRequested(); + + // Regenerate files + if (codeFlowParameters.TpnTemplatePath != null) + { + await _thirdPartyNoticesGenerator.UpdateThirdPartyNotices(codeFlowParameters.TpnTemplatePath); + await _localGitClient.StageAsync(_vmrInfo.VmrPath, [ VmrInfo.ThirdPartyNoticesFileName ], cancellationToken); + } + + if (codeFlowParameters.GenerateCodeOwners) + { + await _codeownersGenerator.UpdateCodeowners(cancellationToken); + } + + if (codeFlowParameters.GenerateCredScanSuppressions) + { + await _credScanSuppressionsGenerator.UpdateCredScanSuppressions(cancellationToken); + } + + var commitMessage = string.Format(RemovalCommitMessage, mapping.Name); + await CommitAsync(commitMessage); + + await workBranch.RebaseAsync(cancellationToken); + + _logger.LogInformation("Repo {repo} removed (staged)", mapping.Name); + } + catch + { + _logger.LogWarning( + InterruptedSyncExceptionMessage, + workBranch.OriginalBranchName.StartsWith("remove") ? "the original" : workBranch.OriginalBranchName); + throw; + } + } + + private async Task RemoveSourceMappingAsync( + string repoName, + LocalPath sourceMappingsPath, + CancellationToken cancellationToken) + { + // Read the existing source-mappings.json file + var json = await _fileSystem.ReadAllTextAsync(sourceMappingsPath); + + var options = new JsonSerializerOptions + { + AllowTrailingCommas = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + ReadCommentHandling = JsonCommentHandling.Skip, + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault | JsonIgnoreCondition.WhenWritingNull, + }; + + var sourceMappingFile = JsonSerializer.Deserialize(json, options) + ?? throw new Exception($"Failed to deserialize {VmrInfo.SourceMappingsFileName}"); + + // Find and remove the mapping + var mappingToRemove = sourceMappingFile.Mappings.FirstOrDefault(m => m.Name == repoName); + if (mappingToRemove == null) + { + _logger.LogWarning("Source mapping for '{repoName}' not found in {file}", repoName, VmrInfo.SourceMappingsFileName); + return false; + } + + sourceMappingFile.Mappings.Remove(mappingToRemove); + + // Write the updated source-mappings.json file + var updatedJson = JsonSerializer.Serialize(sourceMappingFile, options); + _fileSystem.WriteToFile(sourceMappingsPath, updatedJson); + + _logger.LogInformation("Removed source mapping for '{repoName}' from {file}", + repoName, VmrInfo.SourceMappingsFileName); + + return true; + } +} diff --git a/test/Microsoft.DotNet.DarcLib.Codeflow.Tests/CodeFlowTestsBase.cs b/test/Microsoft.DotNet.DarcLib.Codeflow.Tests/CodeFlowTestsBase.cs index d99d8f21e0..ca0514624a 100644 --- a/test/Microsoft.DotNet.DarcLib.Codeflow.Tests/CodeFlowTestsBase.cs +++ b/test/Microsoft.DotNet.DarcLib.Codeflow.Tests/CodeFlowTestsBase.cs @@ -179,6 +179,7 @@ private async Task CallDarcInitialize(string mapping, string commit, LocalPath s await vmrInitializer.InitializeRepository( mappingName: mapping, targetRevision: commit, + remoteUri: null, sourceMappingsPath: sourceMappingsPath, new CodeFlowParameters( AdditionalRemotes: [], @@ -186,6 +187,8 @@ await vmrInitializer.InitializeRepository( GenerateCodeOwners: false, GenerateCredScanSuppressions: false), cancellationToken: _cancellationToken.Token); + + await GitOperations.CommitAll(VmrPath, $"Initialize {mapping} at {commit}"); } protected async Task CallDarcUpdate(string mapping, string commit, bool generateCodeowners = false, bool generateCredScanSuppressions = false)