From ad60f7870ae9aba5012cd3dde085e077aec6043a Mon Sep 17 00:00:00 2001 From: mateipopa Date: Mon, 23 Dec 2024 10:41:44 +0200 Subject: [PATCH] CICD improvements --- .github/workflows/docker-build.yml | 56 ++++++++++++++++++++----- clients/consensus/lighthouse/Dockerfile | 5 ++- clients/exec/erigon/Dockerfile | 5 ++- clients/exec/reth/Dockerfile | 8 ++-- ethereum/ethereum-erigon/Dockerfile | 15 ++++--- ethereum/ethereum-reth/Dockerfile | 14 ++++--- node-base/Dockerfile | 4 +- 7 files changed, 77 insertions(+), 30 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index fe06b63d..f6c005d0 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -74,6 +74,20 @@ jobs: SHA=$(git rev-parse --short HEAD) echo "image_name=node-base" >> $GITHUB_OUTPUT echo "image_tag=${SHA}" >> $GITHUB_OUTPUT + - name: Generate build arguments + id: build-args + run: | + # Extract all ARG declarations that end with _IMAGE from Dockerfile + BUILD_ARGS=$(grep "^ARG.*_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + # Extract the variable name (everything before =) + var_name=$(echo "$line" | sed 's/ARG \(.*\)=.*/\1/') + # Construct the full image path with version + echo "${var_name}=ghcr.io/blockjoy/${var_name/_IMAGE/}:${{ steps.version.outputs.image_tag }}" + done | tr '\n' ',' | sed 's/,$//') + + # Add the standard build args + BUILD_ARGS="${BUILD_ARGS},GRAFANA_LOKI_API_KEY=${{ secrets.GRAFANA_LOKI_API_KEY }},GRAFANA_PROM_API_KEY=${{ secrets.GRAFANA_PROM_API_KEY }}" + echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push base image @@ -81,9 +95,7 @@ jobs: with: context: ./${{ matrix.image_path }} push: true - build-args: | - GRAFANA_LOKI_API_KEY=${{ secrets.GRAFANA_LOKI_API_KEY }} - GRAFANA_PROM_API_KEY=${{ secrets.GRAFANA_PROM_API_KEY }} + build-args: ${{ steps.build-args.outputs.args }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} @@ -125,6 +137,20 @@ jobs: fi echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT + - name: Generate build arguments + id: build-args + run: | + # Extract all ARG declarations that end with _IMAGE from Dockerfile + BUILD_ARGS=$(grep "^ARG.*_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + # Extract the variable name (everything before =) + var_name=$(echo "$line" | sed 's/ARG \(.*\)=.*/\1/') + # Construct the full image path with version + echo "${var_name}=ghcr.io/blockjoy/${var_name/_IMAGE/}:${{ steps.version.outputs.image_tag }}" + done | tr '\n' ',' | sed 's/,$//') + + # Add the standard build args + BUILD_ARGS="${BUILD_ARGS},GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }},GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }},CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }}" + echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push client images @@ -132,10 +158,7 @@ jobs: with: context: ./${{ matrix.image_path }} push: true - build-args: | - GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }} - GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }} - CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }} + build-args: ${{ steps.build-args.outputs.args }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} @@ -164,6 +187,20 @@ jobs: IMAGE_NAME=$(basename $(dirname ${{ matrix.image_path }}))"-"$(basename ${{ matrix.image_path }}) echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT echo "image_tag=${SHA}" >> $GITHUB_OUTPUT + - name: Generate build arguments + id: build-args + run: | + # Extract all ARG declarations that end with _IMAGE from Dockerfile + BUILD_ARGS=$(grep "^ARG.*_IMAGE=" ./${{ matrix.image_path }}/Dockerfile | while read -r line; do + # Extract the variable name (everything before =) + var_name=$(echo "$line" | sed 's/ARG \(.*\)=.*/\1/') + # Construct the full image path with version + echo "${var_name}=ghcr.io/blockjoy/${var_name/_IMAGE/}:${{ steps.version.outputs.image_tag }}" + done | tr '\n' ',' | sed 's/,$//') + + # Add the standard build args + BUILD_ARGS="${BUILD_ARGS},GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }},GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }},CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }}" + echo "args=${BUILD_ARGS}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build and push protocol images @@ -171,10 +208,7 @@ jobs: with: context: ./${{ matrix.image_path }} push: true - build-args: | - GRAFANA_LOKI_BASICAUTH=${{ secrets.GRAFANA_LOKI_BASICAUTH }} - GRAFANA_PROM_BASICAUTH=${{ secrets.GRAFANA_PROM_BASICAUTH }} - CLOUDFLARE_API_KEY=${{ secrets.CLOUDFLARE_API_KEY }} + build-args: ${{ steps.build-args.outputs.args }} cache-from: type=gha cache-to: type=gha,mode=max tags: ghcr.io/blockjoy/${{ steps.version.outputs.image_name }}:${{ steps.version.outputs.image_tag }} diff --git a/clients/consensus/lighthouse/Dockerfile b/clients/consensus/lighthouse/Dockerfile index d9cff401..a49d3d11 100644 --- a/clients/consensus/lighthouse/Dockerfile +++ b/clients/consensus/lighthouse/Dockerfile @@ -1,4 +1,6 @@ -FROM node-base:latest +ARG BASE_IMAGE=ghcr.io/blockjoy/node-base:57f9ec4 + +FROM ${BASE_IMAGE} ENV DEBIAN_FRONTEND=noninteractive @@ -30,4 +32,3 @@ WORKDIR /root/lighthouse RUN . "$HOME/.cargo/env" && cargo build --release --features portable && \ mkdir -p /root/bin && \ cp /root/lighthouse/target/release/lighthouse /root/bin/lighthouse - diff --git a/clients/exec/erigon/Dockerfile b/clients/exec/erigon/Dockerfile index fa3426bc..ed57313e 100644 --- a/clients/exec/erigon/Dockerfile +++ b/clients/exec/erigon/Dockerfile @@ -1,10 +1,12 @@ -FROM node-base:latest +ARG BASE_IMAGE=ghcr.io/blockjoy/node-base:57f9ec4 # Add build arguments for Cloudflare and Grafana push credentials ARG CLOUDFLARE_API_KEY ARG GRAFANA_LOKI_BASICAUTH ARG GRAFANA_PROM_BASICAUTH +FROM ${BASE_IMAGE} + ENV DEBIAN_FRONTEND=noninteractive ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} @@ -26,4 +28,3 @@ WORKDIR /root/erigon RUN make erigon RUN cp /root/erigon/build/bin/erigon /root/bin/erigon - diff --git a/clients/exec/reth/Dockerfile b/clients/exec/reth/Dockerfile index 56f94be9..276a6ffe 100644 --- a/clients/exec/reth/Dockerfile +++ b/clients/exec/reth/Dockerfile @@ -1,9 +1,11 @@ -FROM node-base:latest +ARG BASE_IMAGE=ghcr.io/blockjoy/node-base:57f9ec4 +ARG RETH_VERSION=v1.1.4 +ARG LIGHTHOUSE_VERSION=v5.3.0 + +FROM ${BASE_IMAGE} ENV DEBIAN_FRONTEND=noninteractive -ENV RETH_VERSION=v1.1.4 -ENV LIGHTHOUSE_VERSION=v5.3.0 ENV CARGO_NET_GIT_FETCH_WITH_CLI=true # Install system dependencies including libclang diff --git a/ethereum/ethereum-erigon/Dockerfile b/ethereum/ethereum-erigon/Dockerfile index 7a10a258..e8c294c4 100644 --- a/ethereum/ethereum-erigon/Dockerfile +++ b/ethereum/ethereum-erigon/Dockerfile @@ -1,9 +1,6 @@ -FROM ethereum-erigon:latest as erigon-build - -FROM lighthouse:latest AS lighthouse-build - -FROM node-base:latest - +ARG ERIGON_IMAGE=erigon:latest +ARG LIGHTHOUSE_IMAGE=lighthouse:latest +ARG BASE_IMAGE=node-base:latest ARG CLOUDFLARE_API_KEY ARG GRAFANA_LOKI_BASICAUTH ARG GRAFANA_PROM_BASICAUTH @@ -12,6 +9,12 @@ ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} +FROM ${ERIGON_IMAGE} as erigon-build + +FROM ${LIGHTHOUSE_IMAGE} AS lighthouse-build + +FROM ${BASE_IMAGE} + RUN mkdir -p /root/bin COPY --from=erigon-build /root/bin/erigon /root/bin/ COPY --from=lighthouse-build /root/bin/lighthouse /root/bin/ diff --git a/ethereum/ethereum-reth/Dockerfile b/ethereum/ethereum-reth/Dockerfile index f16ea9ad..c4a38f70 100644 --- a/ethereum/ethereum-reth/Dockerfile +++ b/ethereum/ethereum-reth/Dockerfile @@ -1,8 +1,6 @@ -FROM reth:latest AS reth-build - -FROM lighthouse:latest AS lighthouse-build - -FROM node-base:latest +ARG RETH_IMAGE=reth:latest +ARG LIGHTHOUSE_IMAGE=lighthouse:latest +ARG BASE_IMAGE=node-base:latest ARG CLOUDFLARE_API_KEY ARG GRAFANA_LOKI_BASICAUTH @@ -12,6 +10,12 @@ ENV CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} ENV GRAFANA_LOKI_BASICAUTH=${GRAFANA_LOKI_BASICAUTH} ENV GRAFANA_PROM_BASICAUTH=${GRAFANA_PROM_BASICAUTH} +FROM ${RETH_IMAGE} AS reth-build + +FROM ${LIGHTHOUSE_IMAGE} AS lighthouse-build + +FROM ${BASE_IMAGE} + RUN mkdir -p /root/bin COPY --from=reth-build /root/bin/reth /root/bin/ COPY --from=lighthouse-build /root/bin/lighthouse /root/bin/ diff --git a/node-base/Dockerfile b/node-base/Dockerfile index 112bb9d0..65a2136c 100644 --- a/node-base/Dockerfile +++ b/node-base/Dockerfile @@ -1,4 +1,6 @@ -FROM debian:bookworm-slim@sha256:a165446a88794db4fec31e35e9441433f9552ae048fb1ed26df352d2b537cb96 +ARG BASE_IMAGE=debian:bookworm-slim@sha256:a165446a88794db4fec31e35e9441433f9552ae048fb1ed26df352d2b537cb96 + +FROM ${BASE_IMAGE} ARG GRAFANA_LOKI_API_KEY ARG GRAFANA_PROM_API_KEY