[main] Update common Docker engineering infrastructure with latest (#1139)

Author: dotnet-docker-bot

eng/common/templates/jobs/build-images.yml

@@ -88,30 +88,36 @@ jobs:
     condition: eq(variables.imageBuilderBuildArgs, '')
     displayName: Initialize Image Builder Build Args
   - powershell: |
+      New-Item -Path $(imageInfoHostDir) -ItemType Directory -Force
+
       # Reference the existing imageBuilderBuildArgs variable as an environment variable rather than injecting it directly
       # with the $(imageBuilderBuildArgs) syntax. This is to avoid issues where the string may contain single quotes $ chars
       # which really mess up assigning to a variable. It would require assigning the string with single quotes but also needing
       # to escape the single quotes that are in the string which would need to be done outside the context of PowerShell. Since
       # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates
       # the environment variable for us.
-      New-Item -Path $(imageInfoHostDir) -ItemType Directory -Force
       $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json"
       if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") {
-        $imageBuilderBuildArgs = "$imageBuilderBuildArgs --registry-override $(acr.server) --repo-prefix $(stagingRepoPrefix) --source-repo-prefix $(mirrorRepoPrefix) --push"
+        $imageBuilderBuildArgs = "$imageBuilderBuildArgs --registry-override $(acr-staging.server) --repo-prefix $(stagingRepoPrefix) --source-repo-prefix $(mirrorRepoPrefix) --push"
+      }
+
+      if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.publicProjectName }}" -and ${env:PUBLIC-MIRROR_SERVER} -ne "") {
+        $imageBuilderBuildArgs = "$imageBuilderBuildArgs --base-override-regex '^(?!mcr\.microsoft\.com)' --base-override-sub '$(public-mirror.server)/'"
       }
 
       # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache
       if ("$(pipelineDisabledCache)" -ne "true" -and $env:NOCACHE -ne "true") {
         $imageBuilderBuildArgs = "$imageBuilderBuildArgs --image-info-source-path $(versionsBasePath)$(imageInfoVersionsPath)"
       }
 
+      echo "imageBuilderBuildArgs: $imageBuilderBuildArgs"
       echo "##vso[task.setvariable variable=imageBuilderBuildArgs]$imageBuilderBuildArgs"
     displayName: Set Image Builder Build Args
   - template: /eng/common/templates/steps/run-imagebuilder.yml@self
     parameters:
       name: BuildImages
       displayName: Build Images
-      serviceConnection: $(acr.serviceConnectionName)
+      serviceConnection: $(acr-staging.serviceConnectionName)
       internalProjectName: ${{ parameters.internalProjectName }}
       dockerClientOS: ${{ parameters.dockerClientOS }}
       args: >
@@ -124,8 +130,8 @@ jobs:
         --retry
         --source-repo $(publicGitRepoUri)
         --digests-out-var 'builtImages'
-        --acr-subscription '$(acr.subscription)'
-        --acr-resource-group '$(acr.resourceGroup)'
+        --acr-subscription '$(acr-staging.subscription)'
+        --acr-resource-group '$(acr-staging.resourceGroup)'
         $(manifestVariables)
         $(imageBuilderBuildArgs)
   - template: /eng/common/templates/steps/publish-artifact.yml@self
@@ -172,7 +178,7 @@ jobs:
         # Manifest tool docs: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/secure-supply-chain/custom-sbom-generation-workflows
         $images -Split ',' | ForEach-Object {
           echo "Generating SBOM for $_";
-          $formattedImageName = $_.Replace('$(acr.server)/$(stagingRepoPrefix)', "").Replace('/', '_').Replace(':', '_');
+          $formattedImageName = $_.Replace('$(acr-staging.server)/$(stagingRepoPrefix)', "").Replace('/', '_').Replace(':', '_');
           $sbomChildDir = "$(sbomDirectory)/$formattedImageName";
           New-Item -Type Directory -Path $sbomChildDir > $null;
           & $dotnetPath "$manifestToolDllPath" `

eng/common/templates/jobs/copy-base-images-staging.yml

@@ -0,0 +1,30 @@
+parameters:
+- name: name
+  type: string
+  default: null
+- name: pool
+  type: object
+  default: {}
+- name: customInitSteps
+  type: stepList
+  default: []
+- name: additionalOptions
+  type: string
+  default: ''
+- name: continueOnError
+  type: string
+  default: false
+
+jobs:
+- template: /eng/common/templates/jobs/copy-base-images.yml@self
+  parameters:
+    name: ${{ parameters.name }}
+    pool: ${{ parameters.pool }}
+    customInitSteps: ${{ parameters.customInitSteps }}
+    additionalOptions: ${{ parameters.additionalOptions }}
+    acr:
+      server: $(acr-staging.server)
+      serviceConnection: $(acr-staging.serviceConnectionName)
+      subscription: $(acr-staging.subscription)
+      resourceGroup: $(acr-staging.resourceGroup)
+    repoPrefix: $(mirrorRepoPrefix)

eng/common/templates/jobs/copy-base-images.yml

@@ -1,10 +1,28 @@
 parameters:
-  name: null
-  pool: {}
-  additionalOptions: null
-  publicProjectName: null
-  internalProjectName: null
-  customInitSteps: []
+- name: name
+  type: string
+  default: null
+- name: pool
+  type: object
+  default: {}
+- name: acr
+  type: object
+  default: null
+- name: repoPrefix
+  type: string
+  default: null
+- name: customInitSteps
+  type: stepList
+  default: []
+- name: additionalOptions
+  type: string
+  default: ''
+- name: continueOnError
+  type: string
+  default: false
+- name: forceDryRun
+  type: boolean
+  default: false
 
 jobs:
 - job: ${{ parameters.name }}
@@ -14,7 +32,8 @@ jobs:
   - ${{ parameters.customInitSteps }}
   - template: /eng/common/templates/steps/copy-base-images.yml@self
     parameters:
+      acr: ${{ parameters.acr }} 
+      repoPrefix: ${{ parameters.repoPrefix }}
       additionalOptions: ${{ parameters.additionalOptions }}
-      publicProjectName: ${{ parameters.publicProjectName }}
-      internalProjectName: ${{ parameters.internalProjectName }}
-      continueOnError: true
+      continueOnError: ${{ parameters.continueOnError }}
+      forceDryRun: ${{ parameters.forceDryRun }}

eng/common/templates/jobs/publish.yml

@@ -74,6 +74,7 @@ jobs:
         '$(acr.subscription)'
         '$(acr.resourceGroup)'
         '$(stagingRepoPrefix)'
+        '$(acr-staging.server)'
         --os-type '*'
         --architecture '*'
         --repo-prefix '$(publishRepoPrefix)'

eng/common/templates/stages/build-test-publish-repo.yml

@@ -64,13 +64,11 @@ stages:
             echo "##vso[task.setvariable variable=osVersions]"
             echo "##vso[task.setvariable variable=architecture]"
           displayName: Initialize Test Variables
-  - template: /eng/common/templates/jobs/copy-base-images.yml@self
+  - template: /eng/common/templates/jobs/copy-base-images-staging.yml@self
     parameters:
       name: CopyBaseImages
       pool: ${{ parameters.linuxAmd64Pool }}
       additionalOptions: "--manifest '$(manifest)' $(imageBuilder.pathArgs) $(manifestVariables)"
-      publicProjectName: ${{ parameters.publicProjectName }}
-      internalProjectName: ${{ parameters.internalProjectName }}
       customInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }}
   - template: /eng/common/templates/jobs/generate-matrix.yml@self
     parameters:

eng/common/templates/steps/clean-acr-images.yml

@@ -1,21 +1,25 @@
 parameters:
   repo: null
+  subscription: null
+  resourceGroup: null
+  acr: null
   action: null
   age: null
   customArgs: ""
+  serviceConnection: null
   internalProjectName: null
 steps:
   - template: /eng/common/templates/steps/run-imagebuilder.yml@self
     parameters:
       displayName: Clean ACR Images - ${{ parameters.repo }}
-      serviceConnection: $(acr.serviceConnectionName)
+      serviceConnection: ${{ parameters.serviceConnection }}
       internalProjectName: ${{ parameters.internalProjectName }}
       args: >
         cleanAcrImages
         ${{ parameters.repo }}
-        $(acr.subscription)
-        $(acr.resourceGroup)
-        $(acr.server)
+        ${{ parameters.subscription }}
+        ${{ parameters.resourceGroup }}
+        ${{ parameters.acr }}
         --action ${{ parameters.action }}
         --age ${{ parameters.age }}
         ${{ parameters.customArgs }}

eng/common/templates/steps/copy-base-images-staging.yml

@@ -0,0 +1,19 @@
+parameters:
+- name: additionalOptions
+  type: string
+  default: null
+- name: continueOnError
+  type: string
+  default: false
+
+steps:
+- template: /eng/common/templates/steps/copy-base-images.yml@self
+  parameters:
+    additionalOptions: ${{ parameters.additionalOptions }}
+    continueOnError: ${{ parameters.continueOnError }}
+    repoPrefix: $(mirrorRepoPrefix)
+    acr:
+      resourceGroup: $(acr-staging.resourceGroup)
+      server: $(acr-staging.server)
+      serviceConnection: $(acr-staging.serviceConnectionName)
+      subscription: $(acr-staging.subscription)

eng/common/templates/steps/copy-base-images.yml

@@ -1,29 +1,44 @@
 parameters:
-  additionalOptions: null
-  publicProjectName: null
-  internalProjectName: null
-  continueOnError: false
+- name: acr
+  type: object
+  default:
+    server: ""
+    serviceConnection: ""
+    subscription: ""
+    resourceGroup: ""
+- name: repoPrefix
+  type: string
+  default: null
+- name: additionalOptions
+  type: string
+  default: ""
+- name: continueOnError
+  type: string
+  default: false
+- name: forceDryRun
+  type: boolean
+  default: false
 
 steps:
-- ${{ if or(eq(variables['System.TeamProject'], parameters.publicProjectName), eq(variables['Build.Reason'], 'PullRequest')) }}:
-  - template: /eng/common/templates/steps/set-dry-run.yml@self
+- ${{ if or(eq(parameters.forceDryRun, true), eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}:
+  - script: echo "##vso[task.setvariable variable=dryRunArg]--dry-run"
 - template: /eng/common/templates/steps/run-imagebuilder.yml@self
   parameters:
     displayName: Copy Base Images
-    serviceConnection: $(acr.serviceConnectionName)
+    serviceConnection: '${{ parameters.acr.serviceConnection }}'
     continueOnError: ${{ parameters.continueOnError }}
-    internalProjectName: ${{ parameters.internalProjectName }}
+    internalProjectName: 'internal'
     # Use environment variable to reference $(dryRunArg). Since $(dryRunArg) might be undefined,
     # PowerShell will treat the Azure Pipelines variable macro syntax as a command and throw an
     # error
     args: >
       copyBaseImages
-      '$(acr.subscription)'
-      '$(acr.resourceGroup)'
+      '${{ parameters.acr.subscription }}'
+      '${{ parameters.acr.resourceGroup }}'
       $(dockerHubRegistryCreds)
       $(customCopyBaseImagesArgs)
-      --repo-prefix $(mirrorRepoPrefix)
-      --registry-override '$(acr.server)'
+      --repo-prefix '${{ parameters.repoPrefix }}'
+      --registry-override '${{ parameters.acr.server }}'
       --os-type 'linux'
       --architecture '*'
       $env:DRYRUNARG

eng/common/templates/steps/test-images-linux-client.yml

@@ -22,7 +22,7 @@ steps:
       optionalTestArgs="$optionalTestArgs -TestCategories pre-build"
     else
       if [ "${{ variables['System.TeamProject'] }}" == "${{ parameters.internalProjectName }}" ] && [ "${{ variables['Build.Reason'] }}" != "PullRequest" ]; then
-        optionalTestArgs="$optionalTestArgs -PullImages -Registry $(acr.server) -RepoPrefix $(stagingRepoPrefix) -ImageInfoPath $(artifactsPath)/image-info.json"
+        optionalTestArgs="$optionalTestArgs -PullImages -Registry $(acr-staging.server) -RepoPrefix $(stagingRepoPrefix) -ImageInfoPath $(artifactsPath)/image-info.json"
       fi
       if [ "$REPOTESTARGS" != "" ]; then
         optionalTestArgs="$optionalTestArgs $REPOTESTARGS"
@@ -45,13 +45,13 @@ steps:
   - template: /eng/common/templates/steps/run-pwsh-with-auth.yml@self
     parameters:
       displayName: Docker login
-      serviceConnection: $(acr.serviceConnectionName)
+      serviceConnection: $(acr-staging.serviceConnectionName)
       condition: and(succeeded(), ${{ parameters.condition }})
       command: >
         $azLoginArgs = '--service-principal --tenant $env:AZURE_TENANT_ID -u $env:AZURE_CLIENT_ID --federated-token $env:AZURE_FEDERATED_TOKEN';
         docker exec -e AZURE_TENANT_ID=$env:tenantId -e AZURE_CLIENT_ID=$env:servicePrincipalId -e AZURE_FEDERATED_TOKEN=$env:idToken $(testRunner.container) pwsh
         -File $(engCommonRelativePath)/Invoke-WithRetry.ps1
-        "az login $azLoginArgs; az acr login -n $(acr.server)"
+        "az login $azLoginArgs; az acr login -n $(acr-staging.server)"
   - ${{ if eq(parameters.preBuildValidation, 'false') }}:
     - template: /eng/common/templates/steps/download-build-artifact.yml@self
       parameters:
@@ -73,7 +73,7 @@ steps:
   displayName: Test Images
   condition: and(succeeded(), ${{ parameters.condition }})
 - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
-  - script: docker exec $(testRunner.container) docker logout $(acr.server)
+  - script: docker exec $(testRunner.container) docker logout $(acr-staging.server)
     displayName: Docker logout
     condition: and(always(), ${{ parameters.condition }})
     continueOnError: true

eng/common/templates/steps/test-images-windows-client.yml

@@ -13,17 +13,17 @@ steps:
   - template: /eng/common/templates/steps/run-pwsh-with-auth.yml@self
     parameters:
       displayName: Docker login
-      serviceConnection: $(acr.serviceConnectionName)
+      serviceConnection: $(acr-staging.serviceConnectionName)
       dockerClientOS: windows
       condition: and(succeeded(), ${{ parameters.condition }})
       command: >
         az login --service-principal --tenant $env:tenantId -u $env:servicePrincipalId --federated-token $env:idToken;
-        $accessToken = $(az acr login -n $(acr.server) --expose-token --query accessToken --output tsv);
-        docker login $(acr.server) -u 00000000-0000-0000-0000-000000000000 -p $accessToken
+        $accessToken = $(az acr login -n $(acr-staging.server) --expose-token --query accessToken --output tsv);
+        docker login $(acr-staging.server) -u 00000000-0000-0000-0000-000000000000 -p $accessToken
 - ${{ parameters.customInitSteps }}
 - powershell: |
     if ("${{ variables['System.TeamProject'] }}" -eq "${{ parameters.internalProjectName }}" -and "${{ variables['Build.Reason'] }}" -ne "PullRequest") {
-      $optionalTestArgs="$optionalTestArgs -PullImages -Registry $env:ACR_SERVER -RepoPrefix $env:STAGINGREPOPREFIX -ImageInfoPath $(artifactsPath)/image-info.json"
+      $optionalTestArgs="$optionalTestArgs -PullImages -Registry ${env:ACR-STAGING_SERVER} -RepoPrefix $env:STAGINGREPOPREFIX -ImageInfoPath $(artifactsPath)/image-info.json"
     }
     if ($env:REPOTESTARGS) {
       $optionalTestArgs += " $env:REPOTESTARGS"
@@ -50,7 +50,7 @@ steps:
   displayName: Test Images
   condition: and(succeeded(), ${{ parameters.condition }})
 - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
-  - script: docker logout $(acr.server)
+  - script: docker logout $(acr-staging.server)
     displayName: Docker logout
     condition: and(always(), ${{ parameters.condition }})
     continueOnError: true