[main] Update common Docker engineering infrastructure with latest

eng/common/templates/jobs/build-images.yml

@@ -13,10 +13,9 @@ parameters:
 
 jobs:
 - job: ${{ parameters.name }}
-  condition: and(${{ parameters.matrix }}, not(canceled()), or(in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), eq(${{ parameters.isInternalServicingValidation }}, 'true')))
+  condition: and(${{ parameters.matrix }}, not(canceled()), in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
   dependsOn:
-  - ${{ if eq(parameters.isInternalServicingValidation, 'false') }}:
-    - PreBuildValidation
+  - PreBuildValidation
   - CopyBaseImages
   - GenerateBuildMatrix
   pool: ${{ parameters.pool }}

eng/common/templates/jobs/post-build.yml

@@ -17,6 +17,8 @@ jobs:
   - template: /eng/common/templates/steps/download-build-artifact.yml@self
     parameters:
       targetPath: $(Build.ArtifactStagingDirectory)
+      # This can fail if no build jobs ran to produce any artifacts
+      continueOnError: true
   - powershell: |
       # Move all image-info artifacts to their own directory
       New-Item -ItemType Directory -Path $(imageInfosHostDir)

eng/common/templates/jobs/publish.yml

@@ -44,6 +44,8 @@ jobs:
     parameters:
       targetPath: $(imageInfoHostDir)
       artifactName: image-info
+      # This can fail in scenarios where no build jobs have run to produce any artifacts
+      continueOnError: true
   - template: /eng/common/templates/steps/set-image-info-path-var.yml@self
     parameters:
       publicSourceBranch: $(publicSourceBranch)
@@ -107,21 +109,22 @@ jobs:
   - script: mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data
     displayName: Create EOL Annotation Data Directory
   - script: >
-      $(runImageBuilderCmd) publishImageInfo
-      '$(imageInfoContainerDir)/image-info.json'
-      '$(gitHubVersionsRepoInfo.userName)'
-      '$(gitHubVersionsRepoInfo.email)'
-      '$(gitHubVersionsRepoInfo.accessToken)'
-      --git-owner '$(gitHubVersionsRepoInfo.org)'
-      --git-repo '$(gitHubVersionsRepoInfo.repo)'
-      --git-branch '$(gitHubVersionsRepoInfo.branch)'
-      --git-path '$(gitHubImageInfoVersionsPath)'
-      --image-info-orig-path '$(artifactsPath)/eol-annotation-data/image-info-old.json'
-      --image-info-update-path '$(artifactsPath)/eol-annotation-data/image-info-new.json'
+      curl -fSL
+      --output $(imageInfoHostDir)/full-image-info-orig.json
+      https://raw.githubusercontent.com/$(gitHubVersionsRepoInfo.org)/$(gitHubVersionsRepoInfo.repo)/refs/heads/$(gitHubVersionsRepoInfo.branch)/$(gitHubImageInfoVersionsPath)
+    condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
+    displayName: Download Latest Image Info
+  - script: >
+      $(runImageBuilderCmd) mergeImageInfo
+      $(imageInfoContainerDir)
+      $(imageInfoContainerDir)/full-image-info-new.json
+      $(manifestVariables)
       $(dryRunArg)
-      $(imageBuilder.commonCmdArgs)
+      --manifest $(manifest)
+      --publish
+      --initial-image-info-path $(imageInfoContainerDir)/full-image-info-orig.json
     condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
-    displayName: Publish Image Info
+    displayName: Merge Image Info
   - template: /eng/common/templates/steps/run-imagebuilder.yml@self
     parameters:
       displayName: Ingest Kusto Image Info
@@ -148,8 +151,8 @@ jobs:
       args: >
         generateEolAnnotationData
         '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json'
-        '$(artifactsPath)/eol-annotation-data/image-info-old.json'
-        '$(artifactsPath)/eol-annotation-data/image-info-new.json'
+        '$(imageInfoContainerDir)/full-image-info-orig.json'
+        '$(imageInfoContainerDir)/full-image-info-new.json'
         '$(acr.server)'
         '$(publishRepoPrefix)'
         $(generateEolAnnotationDataExtraOptions)
@@ -166,6 +169,20 @@ jobs:
     parameters:
       internalProjectName: ${{ parameters.internalProjectName }}
       dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json
+  - script: >
+      $(runImageBuilderCmd) publishImageInfo
+      '$(imageInfoContainerDir)/full-image-info-new.json'
+      '$(gitHubVersionsRepoInfo.userName)'
+      '$(gitHubVersionsRepoInfo.email)'
+      '$(gitHubVersionsRepoInfo.accessToken)'
+      --git-owner '$(gitHubVersionsRepoInfo.org)'
+      --git-repo '$(gitHubVersionsRepoInfo.repo)'
+      --git-branch '$(gitHubVersionsRepoInfo.branch)'
+      --git-path '$(gitHubImageInfoVersionsPath)'
+      $(dryRunArg)
+      $(imageBuilder.commonCmdArgs)
+    condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
+    displayName: Publish Image Info
   - script: >
       $(runImageBuilderCmd) postPublishNotification
       '$(publishNotificationRepoName)'

eng/common/templates/stages/build-test-publish-repo.yml

@@ -51,25 +51,24 @@ stages:
 - stage: Build
   condition: and(succeeded(), contains(variables['stages'], 'build'))
   jobs:
-  - ${{ if eq(parameters.isInternalServicingValidation, 'false') }}:
-    - template: /eng/common/templates/jobs/test-images-linux-client.yml@self
-      parameters:
-        name: PreBuildValidation
-        pool: ${{ parameters.linuxAmd64Pool }}
-        testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }}
-        preBuildValidation: true
-        internalProjectName: ${{ parameters.internalProjectName }}
-        customInitSteps:
-          - ${{ parameters.customTestInitSteps }}
-          # These variables are normally set by the matrix. Since this test job is not generated
-          # by a matrix, we need to set them manually. They can be set to empty values since their
-          # values aren't actually used for the pre-build tests.
-          - powershell: |
-              echo "##vso[task.setvariable variable=productVersion]"
-              echo "##vso[task.setvariable variable=imageBuilderPaths]"
-              echo "##vso[task.setvariable variable=osVersions]"
-              echo "##vso[task.setvariable variable=architecture]"
-            displayName: Initialize Test Variables
+  - template: /eng/common/templates/jobs/test-images-linux-client.yml@self
+    parameters:
+      name: PreBuildValidation
+      pool: ${{ parameters.linuxAmd64Pool }}
+      testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }}
+      preBuildValidation: true
+      internalProjectName: ${{ parameters.internalProjectName }}
+      customInitSteps:
+        - ${{ parameters.customTestInitSteps }}
+        # These variables are normally set by the matrix. Since this test job is not generated
+        # by a matrix, we need to set them manually. They can be set to empty values since their
+        # values aren't actually used for the pre-build tests.
+        - powershell: |
+            echo "##vso[task.setvariable variable=productVersion]"
+            echo "##vso[task.setvariable variable=imageBuilderPaths]"
+            echo "##vso[task.setvariable variable=osVersions]"
+            echo "##vso[task.setvariable variable=architecture]"
+          displayName: Initialize Test Variables
   - template: /eng/common/templates/jobs/copy-base-images-staging.yml@self
     parameters:
       name: CopyBaseImages
@@ -344,7 +343,6 @@ stages:
     condition: "
       and(
         not(canceled()),
-        ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'),
         and(
           contains(variables['stages'], 'publish'),
           or(

eng/common/templates/steps/download-build-artifact.yml

@@ -2,6 +2,7 @@ parameters:
   targetPath: ""
   artifactName: ""
   condition: true
+  continueOnError: false
 
 steps:
 - task: DownloadPipelineArtifact@1
@@ -15,3 +16,4 @@ steps:
     artifactName: ${{ parameters.artifactName }}
   displayName: Download Build Artifact(s)
   condition: and(succeeded(), ${{ parameters.condition }})
+  continueOnError: ${{ parameters.continueOnError }}

eng/common/templates/steps/init-docker-windows.yml

@@ -51,7 +51,7 @@ steps:
         $authedImageBuilderCmds = @(
           '$env:AZURE_TENANT_ID = $env:tenantId'
           '$env:AZURE_CLIENT_ID = $env:servicePrincipalId'
-          '$env:AZURE_FEDERATED_TOKEN_FILE' + " = $tokenHostFilePath"
+          '$env:AZURE_FEDERATED_TOKEN_FILE = ' + "'" + "$tokenHostFilePath" + "'"
           $runImageBuilderCmd
         )
 

eng/common/templates/variables/docker-images.yml

@@ -1,6 +1,6 @@
 variables:
-  imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2570468
+  imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2613921
   imageNames.imageBuilder: $(imageNames.imageBuilderName)
   imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId)
-  imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner2.0-docker-testrunner
+  imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner
   imageNames.testRunner.withrepo: testrunner-withrepo:$(Build.BuildId)-$(System.JobId)

eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1

@@ -18,5 +18,5 @@ $onDockerfilesGenerated = {
 }
 
 & $PSScriptRoot/../common/Invoke-ImageBuilder.ps1 `
-    -ImageBuilderArgs "generateDockerfiles --architecture '*' --os-type '*' --optional-templates $customImageBuilderArgs" `
+    -ImageBuilderArgs "generateDockerfiles --optional-templates $customImageBuilderArgs" `
     -OnCommandExecuted $onDockerfilesGenerated