From 0587b238ba7cb270344436c06c6c5f3fec70a5da Mon Sep 17 00:00:00 2001 From: dotnet-docker-bot <60522487+dotnet-docker-bot@users.noreply.github.com> Date: Thu, 9 Jan 2025 08:22:28 -0800 Subject: [PATCH 1/2] Update common Docker engineering infrastructure with latest --- eng/common/templates/jobs/build-images.yml | 5 +- eng/common/templates/jobs/post-build.yml | 2 + eng/common/templates/jobs/publish.yml | 47 +++++++++++++------ .../stages/build-test-publish-repo.yml | 38 +++++++-------- .../steps/download-build-artifact.yml | 2 + .../templates/steps/init-docker-windows.yml | 2 +- .../templates/variables/docker-images.yml | 4 +- 7 files changed, 59 insertions(+), 41 deletions(-) diff --git a/eng/common/templates/jobs/build-images.yml b/eng/common/templates/jobs/build-images.yml index 46c469334..a1063904d 100644 --- a/eng/common/templates/jobs/build-images.yml +++ b/eng/common/templates/jobs/build-images.yml @@ -13,10 +13,9 @@ parameters: jobs: - job: ${{ parameters.name }} - condition: and(${{ parameters.matrix }}, not(canceled()), or(in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), eq(${{ parameters.isInternalServicingValidation }}, 'true'))) + condition: and(${{ parameters.matrix }}, not(canceled()), in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) dependsOn: - - ${{ if eq(parameters.isInternalServicingValidation, 'false') }}: - - PreBuildValidation + - PreBuildValidation - CopyBaseImages - GenerateBuildMatrix pool: ${{ parameters.pool }} diff --git a/eng/common/templates/jobs/post-build.yml b/eng/common/templates/jobs/post-build.yml index d470a6410..31ad2eff0 100644 --- a/eng/common/templates/jobs/post-build.yml +++ b/eng/common/templates/jobs/post-build.yml @@ -17,6 +17,8 @@ jobs: - template: /eng/common/templates/steps/download-build-artifact.yml@self parameters: targetPath: $(Build.ArtifactStagingDirectory) + # This can fail if no build jobs ran to produce any artifacts + continueOnError: true - powershell: | # Move all image-info artifacts to their own directory New-Item -ItemType Directory -Path $(imageInfosHostDir) diff --git a/eng/common/templates/jobs/publish.yml b/eng/common/templates/jobs/publish.yml index 3920beef7..05538ba41 100644 --- a/eng/common/templates/jobs/publish.yml +++ b/eng/common/templates/jobs/publish.yml @@ -44,6 +44,8 @@ jobs: parameters: targetPath: $(imageInfoHostDir) artifactName: image-info + # This can fail in scenarios where no build jobs have run to produce any artifacts + continueOnError: true - template: /eng/common/templates/steps/set-image-info-path-var.yml@self parameters: publicSourceBranch: $(publicSourceBranch) @@ -107,21 +109,22 @@ jobs: - script: mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data displayName: Create EOL Annotation Data Directory - script: > - $(runImageBuilderCmd) publishImageInfo - '$(imageInfoContainerDir)/image-info.json' - '$(gitHubVersionsRepoInfo.userName)' - '$(gitHubVersionsRepoInfo.email)' - '$(gitHubVersionsRepoInfo.accessToken)' - --git-owner '$(gitHubVersionsRepoInfo.org)' - --git-repo '$(gitHubVersionsRepoInfo.repo)' - --git-branch '$(gitHubVersionsRepoInfo.branch)' - --git-path '$(gitHubImageInfoVersionsPath)' - --image-info-orig-path '$(artifactsPath)/eol-annotation-data/image-info-old.json' - --image-info-update-path '$(artifactsPath)/eol-annotation-data/image-info-new.json' + curl -fSL + --output $(imageInfoHostDir)/full-image-info-orig.json + https://raw.githubusercontent.com/$(gitHubVersionsRepoInfo.org)/$(gitHubVersionsRepoInfo.repo)/refs/heads/$(gitHubVersionsRepoInfo.branch)/$(gitHubImageInfoVersionsPath) + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Download Latest Image Info + - script: > + $(runImageBuilderCmd) mergeImageInfo + $(imageInfoContainerDir) + $(imageInfoContainerDir)/full-image-info-new.json + $(manifestVariables) $(dryRunArg) - $(imageBuilder.commonCmdArgs) + --manifest $(manifest) + --publish + --initial-image-info-path $(imageInfoContainerDir)/full-image-info-orig.json condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) - displayName: Publish Image Info + displayName: Merge Image Info - template: /eng/common/templates/steps/run-imagebuilder.yml@self parameters: displayName: Ingest Kusto Image Info @@ -148,8 +151,8 @@ jobs: args: > generateEolAnnotationData '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json' - '$(artifactsPath)/eol-annotation-data/image-info-old.json' - '$(artifactsPath)/eol-annotation-data/image-info-new.json' + '$(imageInfoContainerDir)/full-image-info-orig.json' + '$(imageInfoContainerDir)/full-image-info-new.json' '$(acr.server)' '$(publishRepoPrefix)' $(generateEolAnnotationDataExtraOptions) @@ -166,6 +169,20 @@ jobs: parameters: internalProjectName: ${{ parameters.internalProjectName }} dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json + - script: > + $(runImageBuilderCmd) publishImageInfo + '$(imageInfoContainerDir)/full-image-info-new.json' + '$(gitHubVersionsRepoInfo.userName)' + '$(gitHubVersionsRepoInfo.email)' + '$(gitHubVersionsRepoInfo.accessToken)' + --git-owner '$(gitHubVersionsRepoInfo.org)' + --git-repo '$(gitHubVersionsRepoInfo.repo)' + --git-branch '$(gitHubVersionsRepoInfo.branch)' + --git-path '$(gitHubImageInfoVersionsPath)' + $(dryRunArg) + $(imageBuilder.commonCmdArgs) + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Publish Image Info - script: > $(runImageBuilderCmd) postPublishNotification '$(publishNotificationRepoName)' diff --git a/eng/common/templates/stages/build-test-publish-repo.yml b/eng/common/templates/stages/build-test-publish-repo.yml index e8309beda..aa29b2ba1 100644 --- a/eng/common/templates/stages/build-test-publish-repo.yml +++ b/eng/common/templates/stages/build-test-publish-repo.yml @@ -51,25 +51,24 @@ stages: - stage: Build condition: and(succeeded(), contains(variables['stages'], 'build')) jobs: - - ${{ if eq(parameters.isInternalServicingValidation, 'false') }}: - - template: /eng/common/templates/jobs/test-images-linux-client.yml@self - parameters: - name: PreBuildValidation - pool: ${{ parameters.linuxAmd64Pool }} - testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} - preBuildValidation: true - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: - - ${{ parameters.customTestInitSteps }} - # These variables are normally set by the matrix. Since this test job is not generated - # by a matrix, we need to set them manually. They can be set to empty values since their - # values aren't actually used for the pre-build tests. - - powershell: | - echo "##vso[task.setvariable variable=productVersion]" - echo "##vso[task.setvariable variable=imageBuilderPaths]" - echo "##vso[task.setvariable variable=osVersions]" - echo "##vso[task.setvariable variable=architecture]" - displayName: Initialize Test Variables + - template: /eng/common/templates/jobs/test-images-linux-client.yml@self + parameters: + name: PreBuildValidation + pool: ${{ parameters.linuxAmd64Pool }} + testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} + preBuildValidation: true + internalProjectName: ${{ parameters.internalProjectName }} + customInitSteps: + - ${{ parameters.customTestInitSteps }} + # These variables are normally set by the matrix. Since this test job is not generated + # by a matrix, we need to set them manually. They can be set to empty values since their + # values aren't actually used for the pre-build tests. + - powershell: | + echo "##vso[task.setvariable variable=productVersion]" + echo "##vso[task.setvariable variable=imageBuilderPaths]" + echo "##vso[task.setvariable variable=osVersions]" + echo "##vso[task.setvariable variable=architecture]" + displayName: Initialize Test Variables - template: /eng/common/templates/jobs/copy-base-images-staging.yml@self parameters: name: CopyBaseImages @@ -344,7 +343,6 @@ stages: condition: " and( not(canceled()), - ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'), and( contains(variables['stages'], 'publish'), or( diff --git a/eng/common/templates/steps/download-build-artifact.yml b/eng/common/templates/steps/download-build-artifact.yml index 3b909d83c..ed68ef5b3 100644 --- a/eng/common/templates/steps/download-build-artifact.yml +++ b/eng/common/templates/steps/download-build-artifact.yml @@ -2,6 +2,7 @@ parameters: targetPath: "" artifactName: "" condition: true + continueOnError: false steps: - task: DownloadPipelineArtifact@1 @@ -15,3 +16,4 @@ steps: artifactName: ${{ parameters.artifactName }} displayName: Download Build Artifact(s) condition: and(succeeded(), ${{ parameters.condition }}) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/steps/init-docker-windows.yml b/eng/common/templates/steps/init-docker-windows.yml index d69667f99..a6fbaead6 100644 --- a/eng/common/templates/steps/init-docker-windows.yml +++ b/eng/common/templates/steps/init-docker-windows.yml @@ -51,7 +51,7 @@ steps: $authedImageBuilderCmds = @( '$env:AZURE_TENANT_ID = $env:tenantId' '$env:AZURE_CLIENT_ID = $env:servicePrincipalId' - '$env:AZURE_FEDERATED_TOKEN_FILE' + " = $tokenHostFilePath" + '$env:AZURE_FEDERATED_TOKEN_FILE = ' + "'" + "$tokenHostFilePath" + "'" $runImageBuilderCmd ) diff --git a/eng/common/templates/variables/docker-images.yml b/eng/common/templates/variables/docker-images.yml index a1f9883bf..74f278832 100644 --- a/eng/common/templates/variables/docker-images.yml +++ b/eng/common/templates/variables/docker-images.yml @@ -1,6 +1,6 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2570468 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2613921 imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) - imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner2.0-docker-testrunner + imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner imageNames.testRunner.withrepo: testrunner-withrepo:$(Build.BuildId)-$(System.JobId) From 83d41cc85bc44ccbb1fdfc1ebdb5753e45c81c99 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 10 Jan 2025 08:48:08 -0800 Subject: [PATCH 2/2] Remove platform filters from generate dockerfiles script --- eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1 b/eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1 index 0ac72e8b2..44a810cca 100644 --- a/eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1 +++ b/eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1 @@ -18,5 +18,5 @@ $onDockerfilesGenerated = { } & $PSScriptRoot/../common/Invoke-ImageBuilder.ps1 ` - -ImageBuilderArgs "generateDockerfiles --architecture '*' --os-type '*' --optional-templates $customImageBuilderArgs" ` + -ImageBuilderArgs "generateDockerfiles --optional-templates $customImageBuilderArgs" ` -OnCommandExecuted $onDockerfilesGenerated