Add darc vmr add-repo and remove-repo commands

src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/AddRepoOperation.cs

@@ -0,0 +1,48 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.DotNet.Darc.Options.VirtualMonoRepo;
+using Microsoft.DotNet.DarcLib.Helpers;
+using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo;
+using Microsoft.DotNet.DarcLib.VirtualMonoRepo;
+using Microsoft.Extensions.Logging;
+
+#nullable enable
+namespace Microsoft.DotNet.Darc.Operations.VirtualMonoRepo;
+
+internal class AddRepoOperation : VmrOperationBase
+{
+    private readonly AddRepoCommandLineOptions _options;
+    private readonly IVmrInitializer _vmrInitializer;
+
+    public AddRepoOperation(
+        AddRepoCommandLineOptions options,
+        IVmrInitializer vmrInitializer,
+        ILogger<AddRepoOperation> logger)
+        : base(options, logger)
+    {
+        _options = options;
+        _vmrInitializer = vmrInitializer;
+    }
+
+    protected override async Task ExecuteInternalAsync(
+        string repoName,
+        string? targetRevision,
+        IReadOnlyCollection<AdditionalRemote> additionalRemotes,
+        CancellationToken cancellationToken)
+    {
+        await _vmrInitializer.InitializeRepository(
+            repoName,
+            targetRevision,
+            new NativePath(_options.SourceMappings),
+            new CodeFlowParameters(
+                additionalRemotes,
+                _options.TpnTemplate,
+                _options.GenerateCodeowners,
+                _options.GenerateCredScanSuppressions),
+            cancellationToken);
+    }
+}

src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/RemoveRepoOperation.cs

@@ -0,0 +1,45 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.DotNet.Darc.Options.VirtualMonoRepo;
+using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo;
+using Microsoft.DotNet.DarcLib.VirtualMonoRepo;
+using Microsoft.Extensions.Logging;
+
+#nullable enable
+namespace Microsoft.DotNet.Darc.Operations.VirtualMonoRepo;
+
+internal class RemoveRepoOperation : VmrOperationBase
+{
+    private readonly RemoveRepoCommandLineOptions _options;
+    private readonly IVmrRemover _vmrRemover;
+
+    public RemoveRepoOperation(
+        RemoveRepoCommandLineOptions options,
+        IVmrRemover vmrRemover,
+        ILogger<RemoveRepoOperation> logger)
+        : base(options, logger)
+    {
+        _options = options;
+        _vmrRemover = vmrRemover;
+    }
+
+    protected override async Task ExecuteInternalAsync(
+        string repoName,
+        string? targetRevision,
+        IReadOnlyCollection<AdditionalRemote> additionalRemotes,
+        CancellationToken cancellationToken)
+    {
+        await _vmrRemover.RemoveRepository(
+            repoName,
+            new CodeFlowParameters(
+                additionalRemotes,
+                _options.TpnTemplate,
+                _options.GenerateCodeowners,
+                _options.GenerateCredScanSuppressions),
+            cancellationToken);
+    }
+}

src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/AddRepoCommandLineOptions.cs

@@ -0,0 +1,36 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using CommandLine;
+using Microsoft.DotNet.Darc.Operations.VirtualMonoRepo;
+using Microsoft.DotNet.DarcLib.VirtualMonoRepo;
+
+namespace Microsoft.DotNet.Darc.Options.VirtualMonoRepo;
+
+[Verb("add-repo", HelpText = "Adds new repo(s) to the VMR that haven't been synchronized yet.")]
+internal class AddRepoCommandLineOptions : VmrCommandLineOptions<AddRepoOperation>, IBaseVmrCommandLineOptions
+{
+    [Option("source-mappings", Required = true, HelpText = $"A path to the {VmrInfo.SourceMappingsFileName} file to be used for syncing.")]
+    public string SourceMappings { get; set; }
+
+    [Option("additional-remotes", Required = false, HelpText =
+        "List of additional remote URIs to add to mappings in the format [mapping name]:[remote URI]. " +
+        "Example: installer:https://github.com/myfork/installer sdk:/local/path/to/sdk")]
+    [RedactFromLogging]
+    public IEnumerable<string> AdditionalRemotes { get; set; }
+
+    [Value(0, Required = true, HelpText =
+        "Repository names in the form of NAME or NAME:REVISION where REVISION is a commit SHA or other git reference (branch, tag). " +
+        "Omitting REVISION will synchronize the repo to current HEAD.")]
+    public IEnumerable<string> Repositories { get; set; }
+
+    [Option("tpn-template", Required = false, HelpText = "Path to a template for generating VMRs THIRD-PARTY-NOTICES file. Leave empty to skip generation.")]
+    public string TpnTemplate { get; set; }
+
+    [Option("generate-codeowners", Required = false, HelpText = "Generate a common CODEOWNERS file for all repositories.")]
+    public bool GenerateCodeowners { get; set; } = false;
+
+    [Option("generate-credscansuppressions", Required = false, HelpText = "Generate a common .config/CredScanSuppressions.json file for all repositories.")]
+    public bool GenerateCredScanSuppressions { get; set; } = false;
+}

src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/RemoveRepoCommandLineOptions.cs

@@ -0,0 +1,28 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using CommandLine;
+using Microsoft.DotNet.Darc.Operations.VirtualMonoRepo;
+
+namespace Microsoft.DotNet.Darc.Options.VirtualMonoRepo;
+
+[Verb("remove-repo", HelpText = "Removes repo(s) from the VMR.")]
+internal class RemoveRepoCommandLineOptions : VmrCommandLineOptions<RemoveRepoOperation>, IBaseVmrCommandLineOptions
+{
+    [Value(0, Required = true, HelpText = "Repository names to remove from the VMR.")]
+    public IEnumerable<string> Repositories { get; set; }
+
+    [Option("additional-remotes", Required = false, Hidden = true)]
+    [RedactFromLogging]
+    public IEnumerable<string> AdditionalRemotes { get; set; }
+
+    [Option("tpn-template", Required = false, HelpText = "Path to a template for regenerating VMRs THIRD-PARTY-NOTICES file. Leave empty to skip generation.")]
+    public string TpnTemplate { get; set; }
+
+    [Option("generate-codeowners", Required = false, HelpText = "Regenerate the common CODEOWNERS file for all repositories.")]
+    public bool GenerateCodeowners { get; set; } = false;
+
+    [Option("generate-credscansuppressions", Required = false, HelpText = "Regenerate the common .config/CredScanSuppressions.json file for all repositories.")]
+    public bool GenerateCredScanSuppressions { get; set; } = false;
+}

src/Microsoft.DotNet.Darc/Darc/Program.cs

@@ -145,6 +145,8 @@ public static Type[] GetOptions() =>
     // These are under the "vmr" subcommand
     public static Type[] GetVmrOptions() =>
     [
+        typeof(AddRepoCommandLineOptions),
+        typeof(RemoveRepoCommandLineOptions),
         typeof(InitializeCommandLineOptions),
         typeof(BackflowCommandLineOptions),
         typeof(ForwardFlowCommandLineOptions),

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrRemover.cs

@@ -0,0 +1,23 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo;
+
+#nullable enable
+namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo;
+
+public interface IVmrRemover
+{
+    /// <summary>
+    /// Removes a repository from the VMR.
+    /// </summary>
+    /// <param name="mappingName">Name of a repository mapping</param>
+    /// <param name="codeFlowParameters">Record containing parameters for VMR operations</param>
+    /// <param name="cancellationToken">Cancellation token</param>
+    Task RemoveRepository(
+        string mappingName,
+        CodeFlowParameters codeFlowParameters,
+        CancellationToken cancellationToken);
+}

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs

@@ -111,6 +111,7 @@ private static IServiceCollection AddVmrManagers(
         services.TryAddTransient<IVmrPatchHandler, VmrPatchHandler>();
         services.TryAddTransient<IVmrUpdater, VmrUpdater>();
         services.TryAddTransient<IVmrInitializer, VmrInitializer>();
+        services.TryAddTransient<IVmrRemover, VmrRemover>();
         services.TryAddTransient<IVmrBackFlower, VmrBackFlower>();
         services.TryAddTransient<IVmrForwardFlower, VmrForwardFlower>();
         services.TryAddTransient<ICodeFlowVmrUpdater, CodeFlowVmrUpdater>();

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRemover.cs

@@ -0,0 +1,170 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Maestro.Common;
+using Microsoft.DotNet.DarcLib.Helpers;
+using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo;
+using Microsoft.Extensions.Logging;
+
+#nullable enable
+namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo;
+
+/// <summary>
+/// This class is able to remove a repository from the VMR.
+/// It removes the sources, updates the source manifest, and optionally regenerates 
+/// third-party notices, codeowners, and credential scan suppressions.
+/// </summary>
+public class VmrRemover : VmrManagerBase, IVmrRemover
+{
+    private const string RemovalCommitMessage =
+        $$"""
+        [{name}] Removal of the repository from VMR
+
+        {{Constants.AUTOMATION_COMMIT_TAG}}
+        """;
+
+    private readonly IVmrInfo _vmrInfo;
+    private readonly IVmrDependencyTracker _dependencyTracker;
+    private readonly IWorkBranchFactory _workBranchFactory;
+    private readonly IFileSystem _fileSystem;
+    private readonly ILogger<VmrRemover> _logger;
+    private readonly ISourceManifest _sourceManifest;
+    private readonly ILocalGitClient _localGitClient;
+    private readonly IThirdPartyNoticesGenerator _thirdPartyNoticesGenerator;
+    private readonly ICodeownersGenerator _codeownersGenerator;
+    private readonly ICredScanSuppressionsGenerator _credScanSuppressionsGenerator;
+
+    public VmrRemover(
+        IVmrDependencyTracker dependencyTracker,
+        IVmrPatchHandler patchHandler,
+        IThirdPartyNoticesGenerator thirdPartyNoticesGenerator,
+        ICodeownersGenerator codeownersGenerator,
+        ICredScanSuppressionsGenerator credScanSuppressionsGenerator,
+        ILocalGitClient localGitClient,
+        ILocalGitRepoFactory localGitRepoFactory,
+        IWorkBranchFactory workBranchFactory,
+        IFileSystem fileSystem,
+        ILogger<VmrRemover> logger,
+        ILogger<VmrUpdater> baseLogger,
+        ISourceManifest sourceManifest,
+        IVmrInfo vmrInfo)
+        : base(vmrInfo, dependencyTracker, patchHandler, thirdPartyNoticesGenerator, codeownersGenerator, credScanSuppressionsGenerator, localGitClient, localGitRepoFactory, baseLogger)
+    {
+        _vmrInfo = vmrInfo;
+        _dependencyTracker = dependencyTracker;
+        _workBranchFactory = workBranchFactory;
+        _fileSystem = fileSystem;
+        _logger = logger;
+        _sourceManifest = sourceManifest;
+        _localGitClient = localGitClient;
+        _thirdPartyNoticesGenerator = thirdPartyNoticesGenerator;
+        _codeownersGenerator = codeownersGenerator;
+        _credScanSuppressionsGenerator = credScanSuppressionsGenerator;
+    }
+
+    public async Task RemoveRepository(
+        string mappingName,
+        CodeFlowParameters codeFlowParameters,
+        CancellationToken cancellationToken)
+    {
+        await _dependencyTracker.RefreshMetadataAsync();
+        var mapping = _dependencyTracker.GetMapping(mappingName);
+
+        if (_dependencyTracker.GetDependencyVersion(mapping) is null)
+        {
+            throw new Exception($"Repository {mapping.Name} does not exist in the VMR");
+        }
+
+        var workBranchName = $"remove/{mapping.Name}";
+        IWorkBranch workBranch = await _workBranchFactory.CreateWorkBranchAsync(GetLocalVmr(), workBranchName);
+
+        try
+        {
+            _logger.LogInformation("Removing {name} from the VMR..", mapping.Name);
+
+            var sourcesPath = _vmrInfo.GetRepoSourcesPath(mapping);
+            if (_fileSystem.DirectoryExists(sourcesPath))
+            {
+                _logger.LogInformation("Removing source directory {path}", sourcesPath);
+                _fileSystem.DeleteDirectory(sourcesPath, recursive: true);
+            }
+            else
+            {
+                _logger.LogWarning("Source directory {path} does not exist", sourcesPath);
+            }
+
+            // Remove from source manifest
+            _sourceManifest.RemoveRepository(mapping.Name);
+            _fileSystem.WriteToFile(_vmrInfo.SourceManifestPath, _sourceManifest.ToJson());
+
+            var filesToStage = new List<string>
+            {
+                _vmrInfo.SourceManifestPath,
+                sourcesPath
+            };
+
+            await _localGitClient.StageAsync(_vmrInfo.VmrPath, filesToStage, cancellationToken);
+
+            cancellationToken.ThrowIfCancellationRequested();
+
+            // Regenerate optional files
+            if (codeFlowParameters.TpnTemplatePath != null)
+            {
+                await RegenerateThirdPartyNoticesAsync(codeFlowParameters.TpnTemplatePath, cancellationToken);
+            }
+
+            if (codeFlowParameters.GenerateCodeOwners)
+            {
+                await RegenerateCodeownersAsync(cancellationToken);
+            }
+
+            if (codeFlowParameters.GenerateCredScanSuppressions)
+            {
+                await RegenerateCredScanSuppressionsAsync(cancellationToken);
+            }
+
+            var commitMessage = PrepareCommitMessage(RemovalCommitMessage, mapping.Name);
+            await CommitAsync(commitMessage);
+
+            await workBranch.MergeBackAsync(commitMessage);
+
+            _logger.LogInformation("Removal of {repo} finished", mapping.Name);
+        }
+        catch (Exception)
+        {
+            _logger.LogWarning(
+                InterruptedSyncExceptionMessage,
+                workBranch.OriginalBranchName.StartsWith("sync") || workBranch.OriginalBranchName.StartsWith("init") || workBranch.OriginalBranchName.StartsWith("remove") ?
+                "the original" : workBranch.OriginalBranchName);
+            throw;
+        }
+    }
+
+    private string PrepareCommitMessage(string template, string repoName)
+    {
+        return template.Replace("{name}", repoName);
+    }
+
+    private async Task RegenerateThirdPartyNoticesAsync(string templatePath, CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("Regenerating third-party notices");
+        await _thirdPartyNoticesGenerator.UpdateThirdPartyNotices(templatePath);
+        await _localGitClient.StageAsync(_vmrInfo.VmrPath, new[] { VmrInfo.ThirdPartyNoticesFileName }, cancellationToken);
+    }
+
+    private async Task RegenerateCodeownersAsync(CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("Regenerating CODEOWNERS");
+        await _codeownersGenerator.UpdateCodeowners(cancellationToken);
+    }
+
+    private async Task RegenerateCredScanSuppressionsAsync(CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("Regenerating credential scan suppressions");
+        await _credScanSuppressionsGenerator.UpdateCredScanSuppressions(cancellationToken);
+    }
+}