From 754d0fdacc466c58e34967e5fb4961dcfe4762bc Mon Sep 17 00:00:00 2001 From: mkosiarc Date: Thu, 28 Nov 2024 10:12:23 +0100 Subject: [PATCH] Improve generation of base images SBOMs most functional changes are in the related PR that updates the base_images_sbom_script.py https://github.com/konflux-ci/build-tasks-dockerfiles/pull/191 Here, we are just updating on how we generate the inputs for this script. We are now passing the whole parsed Dockerfile in json format to that script, which allows us to better parse/detect base images. Also, the format of the /shared/base_images_digests file was changed. Previously we could rely on the order of the image references with the digests in the file. Now we need to provide a mapping from an image reference as it was used in the Dockerfile to the full image reference with digests that was used during build and generated by buildah. The mapping is done as: KFLUXBUGS-1718 Signed-off-by: mkosiarc --- task/buildah-oci-ta/0.2/buildah-oci-ta.yaml | 14 ++++++-------- .../0.2/buildah-remote-oci-ta.yaml | 14 ++++++-------- task/buildah-remote/0.2/buildah-remote.yaml | 15 +++++++-------- task/buildah/0.2/buildah.yaml | 13 ++++++------- 4 files changed, 25 insertions(+), 31 deletions(-) diff --git a/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml b/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml index 7a0ecf4764..8d1caca85f 100644 --- a/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml +++ b/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml @@ -348,9 +348,9 @@ spec: BUILD_ARG_FLAGS+=("--build-arg=$build_arg") done + dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" >/shared/parsed_dockerfile.json BASE_IMAGES=$( - dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" | - jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' + jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' /shared/parsed_dockerfile.json ) BUILDAH_ARGS=() @@ -530,11 +530,9 @@ spec: touch /shared/base_images_digests for image in $BASE_IMAGES; do - buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image" >>/shared/base_images_digests + base_image_digest=$(buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image") + echo "$image $base_image_digest" >>/shared/base_images_digests done - - # Needed to generate base images SBOM - echo "$BASE_IMAGES" >/shared/base_images_from_dockerfile computeResources: limits: cpu: "4" @@ -590,7 +588,7 @@ spec: securityContext: runAsUser: 0 - name: prepare-sboms - image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:11851ba63f63dfdcf722e47993f41a1f5f31a7a0dc8aa85b810ce2466daf23af + image: quay.io/mkosiarc_rhtap/base-images-sbom-script:my-change workingDir: /var/workdir script: | echo "Merging contents of sbom-source.json and sbom-image.json into sbom-cyclonedx.json" @@ -608,7 +606,7 @@ spec: echo "Adding base images data to sbom-cyclonedx.json" python3 /scripts/base_images_sbom_script.py \ --sbom=sbom-cyclonedx.json \ - --base-images-from-dockerfile=/shared/base_images_from_dockerfile \ + --parsed-dockerfile=/shared/parsed_dockerfile.json \ --base-images-digests=/shared/base_images_digests computeResources: limits: diff --git a/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml b/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml index 954bb01d48..8b7199012a 100644 --- a/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml +++ b/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml @@ -382,9 +382,9 @@ spec: BUILD_ARG_FLAGS+=("--build-arg=$build_arg") done + dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" >/shared/parsed_dockerfile.json BASE_IMAGES=$( - dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" | - jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' + jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' /shared/parsed_dockerfile.json ) BUILDAH_ARGS=() @@ -564,12 +564,10 @@ spec: touch /shared/base_images_digests for image in $BASE_IMAGES; do - buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image" >>/shared/base_images_digests + base_image_digest=$(buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image") + echo "$image $base_image_digest" >>/shared/base_images_digests done - # Needed to generate base images SBOM - echo "$BASE_IMAGES" >/shared/base_images_from_dockerfile - buildah push "$IMAGE" "oci:konflux-final-image:$IMAGE" REMOTESSHEOF chmod +x scripts/script-build.sh @@ -704,7 +702,7 @@ spec: requests: cpu: 100m memory: 256Mi - image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:11851ba63f63dfdcf722e47993f41a1f5f31a7a0dc8aa85b810ce2466daf23af + image: quay.io/mkosiarc_rhtap/base-images-sbom-script:my-change name: prepare-sboms script: | #!/bin/bash @@ -728,7 +726,7 @@ spec: echo "Adding base images data to sbom-cyclonedx.json" python3 /scripts/base_images_sbom_script.py \ --sbom=sbom-cyclonedx.json \ - --base-images-from-dockerfile=/shared/base_images_from_dockerfile \ + --parsed-dockerfile=/shared/parsed_dockerfile.json \ --base-images-digests=/shared/base_images_digests securityContext: runAsUser: 0 diff --git a/task/buildah-remote/0.2/buildah-remote.yaml b/task/buildah-remote/0.2/buildah-remote.yaml index 371fac0c1e..2fc80a624d 100644 --- a/task/buildah-remote/0.2/buildah-remote.yaml +++ b/task/buildah-remote/0.2/buildah-remote.yaml @@ -358,9 +358,10 @@ spec: BUILD_ARG_FLAGS+=("--build-arg=$build_arg") done + + dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" > /shared/parsed_dockerfile.json BASE_IMAGES=$( - dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" | - jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' + jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' /shared/parsed_dockerfile.json ) BUILDAH_ARGS=() @@ -542,12 +543,10 @@ spec: touch /shared/base_images_digests for image in $BASE_IMAGES; do - buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image" >> /shared/base_images_digests + base_image_digest=$(buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image") + echo "$image $base_image_digest" >> /shared/base_images_digests done - # Needed to generate base images SBOM - echo "$BASE_IMAGES" > /shared/base_images_from_dockerfile - buildah push "$IMAGE" "oci:konflux-final-image:$IMAGE" REMOTESSHEOF chmod +x scripts/script-build.sh @@ -682,7 +681,7 @@ spec: requests: cpu: 100m memory: 256Mi - image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:11851ba63f63dfdcf722e47993f41a1f5f31a7a0dc8aa85b810ce2466daf23af + image: quay.io/mkosiarc_rhtap/base-images-sbom-script:my-change name: prepare-sboms script: | #!/bin/bash @@ -706,7 +705,7 @@ spec: echo "Adding base images data to sbom-cyclonedx.json" python3 /scripts/base_images_sbom_script.py \ --sbom=sbom-cyclonedx.json \ - --base-images-from-dockerfile=/shared/base_images_from_dockerfile \ + --parsed-dockerfile=/shared/parsed_dockerfile.json \ --base-images-digests=/shared/base_images_digests securityContext: runAsUser: 0 diff --git a/task/buildah/0.2/buildah.yaml b/task/buildah/0.2/buildah.yaml index 9da3a89fc8..96d3c36ddd 100644 --- a/task/buildah/0.2/buildah.yaml +++ b/task/buildah/0.2/buildah.yaml @@ -279,9 +279,10 @@ spec: BUILD_ARG_FLAGS+=("--build-arg=$build_arg") done + + dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" > /shared/parsed_dockerfile.json BASE_IMAGES=$( - dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" | - jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' + jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' /shared/parsed_dockerfile.json ) BUILDAH_ARGS=() @@ -463,12 +464,10 @@ spec: touch /shared/base_images_digests for image in $BASE_IMAGES; do - buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image" >> /shared/base_images_digests + base_image_digest=$(buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image") + echo "$image $base_image_digest" >> /shared/base_images_digests done - # Needed to generate base images SBOM - echo "$BASE_IMAGES" > /shared/base_images_from_dockerfile - securityContext: capabilities: add: @@ -558,7 +557,7 @@ spec: echo "Adding base images data to sbom-cyclonedx.json" python3 /scripts/base_images_sbom_script.py \ --sbom=sbom-cyclonedx.json \ - --base-images-from-dockerfile=/shared/base_images_from_dockerfile \ + --parsed-dockerfile=/shared/parsed_dockerfile.json \ --base-images-digests=/shared/base_images_digests workingDir: $(workspaces.source.path) securityContext: