diff --git a/eng/common/templates/jobs/build-images.yml b/eng/common/templates/jobs/build-images.yml index 46c46933..a1063904 100644 --- a/eng/common/templates/jobs/build-images.yml +++ b/eng/common/templates/jobs/build-images.yml @@ -13,10 +13,9 @@ parameters: jobs: - job: ${{ parameters.name }} - condition: and(${{ parameters.matrix }}, not(canceled()), or(in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), eq(${{ parameters.isInternalServicingValidation }}, 'true'))) + condition: and(${{ parameters.matrix }}, not(canceled()), in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) dependsOn: - - ${{ if eq(parameters.isInternalServicingValidation, 'false') }}: - - PreBuildValidation + - PreBuildValidation - CopyBaseImages - GenerateBuildMatrix pool: ${{ parameters.pool }} diff --git a/eng/common/templates/jobs/post-build.yml b/eng/common/templates/jobs/post-build.yml index d470a641..31ad2eff 100644 --- a/eng/common/templates/jobs/post-build.yml +++ b/eng/common/templates/jobs/post-build.yml @@ -17,6 +17,8 @@ jobs: - template: /eng/common/templates/steps/download-build-artifact.yml@self parameters: targetPath: $(Build.ArtifactStagingDirectory) + # This can fail if no build jobs ran to produce any artifacts + continueOnError: true - powershell: | # Move all image-info artifacts to their own directory New-Item -ItemType Directory -Path $(imageInfosHostDir) diff --git a/eng/common/templates/jobs/publish.yml b/eng/common/templates/jobs/publish.yml index 3920beef..05538ba4 100644 --- a/eng/common/templates/jobs/publish.yml +++ b/eng/common/templates/jobs/publish.yml @@ -44,6 +44,8 @@ jobs: parameters: targetPath: $(imageInfoHostDir) artifactName: image-info + # This can fail in scenarios where no build jobs have run to produce any artifacts + continueOnError: true - template: /eng/common/templates/steps/set-image-info-path-var.yml@self parameters: publicSourceBranch: $(publicSourceBranch) @@ -107,21 +109,22 @@ jobs: - script: mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data displayName: Create EOL Annotation Data Directory - script: > - $(runImageBuilderCmd) publishImageInfo - '$(imageInfoContainerDir)/image-info.json' - '$(gitHubVersionsRepoInfo.userName)' - '$(gitHubVersionsRepoInfo.email)' - '$(gitHubVersionsRepoInfo.accessToken)' - --git-owner '$(gitHubVersionsRepoInfo.org)' - --git-repo '$(gitHubVersionsRepoInfo.repo)' - --git-branch '$(gitHubVersionsRepoInfo.branch)' - --git-path '$(gitHubImageInfoVersionsPath)' - --image-info-orig-path '$(artifactsPath)/eol-annotation-data/image-info-old.json' - --image-info-update-path '$(artifactsPath)/eol-annotation-data/image-info-new.json' + curl -fSL + --output $(imageInfoHostDir)/full-image-info-orig.json + https://raw.githubusercontent.com/$(gitHubVersionsRepoInfo.org)/$(gitHubVersionsRepoInfo.repo)/refs/heads/$(gitHubVersionsRepoInfo.branch)/$(gitHubImageInfoVersionsPath) + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Download Latest Image Info + - script: > + $(runImageBuilderCmd) mergeImageInfo + $(imageInfoContainerDir) + $(imageInfoContainerDir)/full-image-info-new.json + $(manifestVariables) $(dryRunArg) - $(imageBuilder.commonCmdArgs) + --manifest $(manifest) + --publish + --initial-image-info-path $(imageInfoContainerDir)/full-image-info-orig.json condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) - displayName: Publish Image Info + displayName: Merge Image Info - template: /eng/common/templates/steps/run-imagebuilder.yml@self parameters: displayName: Ingest Kusto Image Info @@ -148,8 +151,8 @@ jobs: args: > generateEolAnnotationData '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json' - '$(artifactsPath)/eol-annotation-data/image-info-old.json' - '$(artifactsPath)/eol-annotation-data/image-info-new.json' + '$(imageInfoContainerDir)/full-image-info-orig.json' + '$(imageInfoContainerDir)/full-image-info-new.json' '$(acr.server)' '$(publishRepoPrefix)' $(generateEolAnnotationDataExtraOptions) @@ -166,6 +169,20 @@ jobs: parameters: internalProjectName: ${{ parameters.internalProjectName }} dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json + - script: > + $(runImageBuilderCmd) publishImageInfo + '$(imageInfoContainerDir)/full-image-info-new.json' + '$(gitHubVersionsRepoInfo.userName)' + '$(gitHubVersionsRepoInfo.email)' + '$(gitHubVersionsRepoInfo.accessToken)' + --git-owner '$(gitHubVersionsRepoInfo.org)' + --git-repo '$(gitHubVersionsRepoInfo.repo)' + --git-branch '$(gitHubVersionsRepoInfo.branch)' + --git-path '$(gitHubImageInfoVersionsPath)' + $(dryRunArg) + $(imageBuilder.commonCmdArgs) + condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + displayName: Publish Image Info - script: > $(runImageBuilderCmd) postPublishNotification '$(publishNotificationRepoName)' diff --git a/eng/common/templates/stages/build-test-publish-repo.yml b/eng/common/templates/stages/build-test-publish-repo.yml index e8309bed..aa29b2ba 100644 --- a/eng/common/templates/stages/build-test-publish-repo.yml +++ b/eng/common/templates/stages/build-test-publish-repo.yml @@ -51,25 +51,24 @@ stages: - stage: Build condition: and(succeeded(), contains(variables['stages'], 'build')) jobs: - - ${{ if eq(parameters.isInternalServicingValidation, 'false') }}: - - template: /eng/common/templates/jobs/test-images-linux-client.yml@self - parameters: - name: PreBuildValidation - pool: ${{ parameters.linuxAmd64Pool }} - testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} - preBuildValidation: true - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: - - ${{ parameters.customTestInitSteps }} - # These variables are normally set by the matrix. Since this test job is not generated - # by a matrix, we need to set them manually. They can be set to empty values since their - # values aren't actually used for the pre-build tests. - - powershell: | - echo "##vso[task.setvariable variable=productVersion]" - echo "##vso[task.setvariable variable=imageBuilderPaths]" - echo "##vso[task.setvariable variable=osVersions]" - echo "##vso[task.setvariable variable=architecture]" - displayName: Initialize Test Variables + - template: /eng/common/templates/jobs/test-images-linux-client.yml@self + parameters: + name: PreBuildValidation + pool: ${{ parameters.linuxAmd64Pool }} + testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} + preBuildValidation: true + internalProjectName: ${{ parameters.internalProjectName }} + customInitSteps: + - ${{ parameters.customTestInitSteps }} + # These variables are normally set by the matrix. Since this test job is not generated + # by a matrix, we need to set them manually. They can be set to empty values since their + # values aren't actually used for the pre-build tests. + - powershell: | + echo "##vso[task.setvariable variable=productVersion]" + echo "##vso[task.setvariable variable=imageBuilderPaths]" + echo "##vso[task.setvariable variable=osVersions]" + echo "##vso[task.setvariable variable=architecture]" + displayName: Initialize Test Variables - template: /eng/common/templates/jobs/copy-base-images-staging.yml@self parameters: name: CopyBaseImages @@ -344,7 +343,6 @@ stages: condition: " and( not(canceled()), - ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'), and( contains(variables['stages'], 'publish'), or( diff --git a/eng/common/templates/steps/download-build-artifact.yml b/eng/common/templates/steps/download-build-artifact.yml index 3b909d83..ed68ef5b 100644 --- a/eng/common/templates/steps/download-build-artifact.yml +++ b/eng/common/templates/steps/download-build-artifact.yml @@ -2,6 +2,7 @@ parameters: targetPath: "" artifactName: "" condition: true + continueOnError: false steps: - task: DownloadPipelineArtifact@1 @@ -15,3 +16,4 @@ steps: artifactName: ${{ parameters.artifactName }} displayName: Download Build Artifact(s) condition: and(succeeded(), ${{ parameters.condition }}) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/steps/init-docker-windows.yml b/eng/common/templates/steps/init-docker-windows.yml index d69667f9..a6fbaead 100644 --- a/eng/common/templates/steps/init-docker-windows.yml +++ b/eng/common/templates/steps/init-docker-windows.yml @@ -51,7 +51,7 @@ steps: $authedImageBuilderCmds = @( '$env:AZURE_TENANT_ID = $env:tenantId' '$env:AZURE_CLIENT_ID = $env:servicePrincipalId' - '$env:AZURE_FEDERATED_TOKEN_FILE' + " = $tokenHostFilePath" + '$env:AZURE_FEDERATED_TOKEN_FILE = ' + "'" + "$tokenHostFilePath" + "'" $runImageBuilderCmd ) diff --git a/eng/common/templates/variables/docker-images.yml b/eng/common/templates/variables/docker-images.yml index a1f9883b..74f27883 100644 --- a/eng/common/templates/variables/docker-images.yml +++ b/eng/common/templates/variables/docker-images.yml @@ -1,6 +1,6 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2570468 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2613921 imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) - imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner2.0-docker-testrunner + imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner imageNames.testRunner.withrepo: testrunner-withrepo:$(Build.BuildId)-$(System.JobId) diff --git a/eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1 b/eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1 index 0ac72e8b..44a810cc 100644 --- a/eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1 +++ b/eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1 @@ -18,5 +18,5 @@ $onDockerfilesGenerated = { } & $PSScriptRoot/../common/Invoke-ImageBuilder.ps1 ` - -ImageBuilderArgs "generateDockerfiles --architecture '*' --os-type '*' --optional-templates $customImageBuilderArgs" ` + -ImageBuilderArgs "generateDockerfiles --optional-templates $customImageBuilderArgs" ` -OnCommandExecuted $onDockerfilesGenerated