Extract common stuff from the different Dockerfiles

All Dockerfiles share most of the steps, apart from a couple of steps at the end. As such, to cut build time, I extracted the common steps into a separate Docker imaeg that all other images build on top of it.
aws · Jan 31, 2024 · d056003 · d056003
1 parent da1df8e
commit d056003
Show file tree

Hide file tree

Showing 13 changed files with 301 additions and 711 deletions.
diff --git a/images/airflow/2.8.0/Dockerfile.j2 → images/airflow/2.8.0/Dockerfile.base.j2 b/images/airflow/2.8.0/Dockerfile.j2 → images/airflow/2.8.0/Dockerfile.base.j2
@@ -94,33 +94,3 @@ WORKDIR ${AIRFLOW_USER_HOME}
 
 # Copy python files.
 COPY ./python /python
-
-{% if build_type == 'standard' %}
-{# This is the standard build type. it is what customer uses.#}
-USER airflow
-
-ENTRYPOINT ["python3", "-m", "mwaa.entrypoint"]
-
-CMD shell
-{% elif build_type == 'explorer' %}
-{#
-The 'explorer' build type is almost identical to the 'standard' build type but
-it doesn't include the entrypoint. This is useful for debugging purposes to run
-the image and look around its content without starting airflow, which might
-require further setup.
-#}
-USER airflow
-
-ENTRYPOINT ["/bin/bash"]
-{% elif build_type == 'explorer-privileged' %}
-{#
-This is similar to the 'explorer' build type, but additionally uses the root
-user, giving the user of this Docker image elevated permissions.  The user can,
-thus, install packages, remove packages, or anything else.
-#}
-USER root
-
-ENTRYPOINT ["/bin/bash"]
-{% else %}
-{{ raise("Invalid build type.") }}
-{% endif %}
diff --git a/images/airflow/2.8.0/Dockerfile.derivatives.j2 b/images/airflow/2.8.0/Dockerfile.derivatives.j2
@@ -0,0 +1,60 @@
+FROM amazon-mwaa/airflow:2.8.0-base
+
+{% if bootstrapping_scripts_dev %}
+
+# Copy bootstrapping files.
+COPY ./bootstrap-dev /bootstrap-dev
+RUN chmod -R +x /bootstrap-dev
+
+{#
+Those steps are only exectued for development images. Those are images that
+contain additional packages that help with debugging, e.g. editors, etc., but
+are not supposed to be in production.
+#}
+
+#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+# BEGINNING marker for dev bootstrapping steps.
+#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+
+{% for filepath in bootstrapping_scripts_dev %}
+RUN {{ filepath }}
+{% endfor %}
+
+#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+# END marker for dev bootstrapping steps.
+#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+
+# remove bootstrapping files.
+RUN rm -rf /bootstrap-dev
+
+{% endif %}
+
+{% if build_type == 'standard' %}
+{# This is the standard build type. it is what customer uses.#}
+USER airflow
+
+ENTRYPOINT ["python3", "-m", "mwaa.entrypoint"]
+
+CMD shell
+{% elif build_type == 'explorer' %}
+{#
+The 'explorer' build type is almost identical to the 'standard' build type but
+it doesn't include the entrypoint. This is useful for debugging purposes to run
+the image and look around its content without starting airflow, which might
+require further setup.
+#}
+USER airflow
+
+ENTRYPOINT ["/bin/bash"]
+{% elif build_type == 'explorer-privileged' %}
+{#
+This is similar to the 'explorer' build type, but additionally uses the root
+user, giving the user of this Docker image elevated permissions.  The user can,
+thus, install packages, remove packages, or anything else.
+#}
+USER root
+
+ENTRYPOINT ["/bin/bash"]
+{% else %}
+{{ raise("Invalid build type.") }}
+{% endif %}
diff --git a/images/airflow/2.8.0/Dockerfiles/Dockerfile b/images/airflow/2.8.0/Dockerfiles/Dockerfile
@@ -3,119 +3,10 @@
 # the Jinja2-templated Dockerfile.j2 file, so you need to change that file
 # instead.
 #
-# This file was generated on 2024-01-30 16:34:46.229453
+# This file was generated on 2024-01-31 02:49:24.657364
 #
 
-FROM public.ecr.aws/amazonlinux/amazonlinux:2023
-
-# Environment variables
-ENV AIRFLOW_AMAZON_PROVIDERS_VERSION=8.13.0
-ENV AIRFLOW_USER_HOME=/usr/local/airflow
-ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME}
-ENV AIRFLOW_VERSION=2.8.0
-ENV MWAA_HOME=/usr/local/mwaa
-ENV PYTHON_VERSION=3.11.7
-
-# We don't want those variables to stay in the final image, so we use ARG instead of ENV.
-ARG PATH_DEFAULT=${PATH}
-ARG PATH_AIRFLOW_USER=${AIRFLOW_USER_HOME}/.local/bin:${PATH_DEFAULT}
-ARG PYTHON_MD5_CHECKSUM=d96c7e134c35a8c46236f8a0e566b69c
-ARG MARIADB_DOWNLOAD_BASE_URL=https://mirror.mariadb.org/yum/11.1/fedora38-amd64/rpms
-ARG MARIADB_RPM_COMMON=MariaDB-common-11.1.2-1.fc38.x86_64.rpm
-ARG MARIADB_RPM_COMMON_CHECKSUM=e87371d558efa97724f3728fb214cf19
-ARG MARIADB_RPM_DEVEL=MariaDB-devel-11.1.2-1.fc38.x86_64.rpm
-ARG MARIADB_RPM_DEVEL_CHECKSUM=cfce6e9b53f4e4fb1cb14f1ed720c92c
-ARG MARIADB_RPM_SHARED=MariaDB-shared-11.1.2-1.fc38.x86_64.rpm
-ARG MARIADB_RPM_SHARED_CHECKSUM=ed82ad5bc5b35cb2719a9471a71c6cdb
-
-# Copy bootstrapping files.
-COPY ./bootstrap /bootstrap
-RUN chmod -R +x /bootstrap
-
-#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
-# BEGINNING marker for root user, first pass bootstrapping steps.
-#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
-
-
-RUN /bootstrap/01-root-firstpass/001-init.sh
-
-RUN /bootstrap/01-root-firstpass/002-install-python.sh
-
-RUN /bootstrap/01-root-firstpass/003-install-mariadb.sh
-
-RUN /bootstrap/01-root-firstpass/004-create-airflow-user.sh
-
-RUN /bootstrap/01-root-firstpass/005-install-aws-cli.sh
-
-RUN /bootstrap/01-root-firstpass/100-install-needed-dnf-packages.sh
-
-
-#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
-# END marker for root user, first pass bootstrapping steps.
-#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
-
-
-
-#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
-# > BEGINNING marker for airflow user bootstrapping steps.
-#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
-
-# Switch to 'airflow' user and update the PATH environment variable.
-USER airflow
-ENV PATH=${PATH_AIRFLOW_USER} 
-
-
-RUN /bootstrap/02-airflow/001-install-airflow.sh
-
-
-# Revert the PATH and user.
-ENV PATH=${PATH_DEFAULT}
-USER root
-
-#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
-# > END marker for airflow user bootstrapping steps.
-#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
-
-
-
-#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
-# BEGINNING marker for root user, second pass bootstrapping steps.
-#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
-
-# Those steps are also executed as the root user. However, they rely on the
-# successfull execution of the airflow user bootstrapping steps. For example,
-# giving ownership of the Airflow home user to the 'airflow' user requires the
-# the folder to be fully setup first.
-
-
-RUN /bootstrap/03-root-secondpass/001-create-mwaa-dir.sh
-
-RUN /bootstrap/03-root-secondpass/100-chown-airflow-folder.sh
-
-
-#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
-# END marker for root user, second pass bootstrapping steps.
-#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
-
-# remove bootstrapping files.
-RUN rm -rf /bootstrap
-
-# Create a volume for syncing files with the sidecar. The actual folder
-# is created by the `001-create-mwaa-dir.sh` script.
-VOLUME ["${MWAA_HOME}"]
-
-# TODO We should only expose this port if the command is 'webserver'.
-EXPOSE 8080
-
-ENV PATH=${PATH_AIRFLOW_USER}
-ENV PYTHONPATH="/python"
-
-WORKDIR ${AIRFLOW_USER_HOME}
-
-# Copy python files.
-COPY ./python /python
-
-
+FROM amazon-mwaa/airflow:2.8.0-base
 
 USER airflow
 

diff --git a/images/airflow/2.8.0/Dockerfiles/Dockerfile-dev b/images/airflow/2.8.0/Dockerfiles/Dockerfile-dev
@@ -3,121 +3,27 @@
 # the Jinja2-templated Dockerfile.j2 file, so you need to change that file
 # instead.
 #
-# This file was generated on 2024-01-30 16:34:46.218794
+# This file was generated on 2024-01-31 02:49:24.649403
 #
 
-FROM public.ecr.aws/amazonlinux/amazonlinux:2023
-
-# Environment variables
-ENV AIRFLOW_AMAZON_PROVIDERS_VERSION=8.13.0
-ENV AIRFLOW_USER_HOME=/usr/local/airflow
-ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME}
-ENV AIRFLOW_VERSION=2.8.0
-ENV MWAA_HOME=/usr/local/mwaa
-ENV PYTHON_VERSION=3.11.7
-
-# We don't want those variables to stay in the final image, so we use ARG instead of ENV.
-ARG PATH_DEFAULT=${PATH}
-ARG PATH_AIRFLOW_USER=${AIRFLOW_USER_HOME}/.local/bin:${PATH_DEFAULT}
-ARG PYTHON_MD5_CHECKSUM=d96c7e134c35a8c46236f8a0e566b69c
-ARG MARIADB_DOWNLOAD_BASE_URL=https://mirror.mariadb.org/yum/11.1/fedora38-amd64/rpms
-ARG MARIADB_RPM_COMMON=MariaDB-common-11.1.2-1.fc38.x86_64.rpm
-ARG MARIADB_RPM_COMMON_CHECKSUM=e87371d558efa97724f3728fb214cf19
-ARG MARIADB_RPM_DEVEL=MariaDB-devel-11.1.2-1.fc38.x86_64.rpm
-ARG MARIADB_RPM_DEVEL_CHECKSUM=cfce6e9b53f4e4fb1cb14f1ed720c92c
-ARG MARIADB_RPM_SHARED=MariaDB-shared-11.1.2-1.fc38.x86_64.rpm
-ARG MARIADB_RPM_SHARED_CHECKSUM=ed82ad5bc5b35cb2719a9471a71c6cdb
+FROM amazon-mwaa/airflow:2.8.0-base
 
 # Copy bootstrapping files.
-COPY ./bootstrap /bootstrap
-RUN chmod -R +x /bootstrap
-
-#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
-# BEGINNING marker for root user, first pass bootstrapping steps.
-#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
-
-
-RUN /bootstrap/01-root-firstpass/001-init.sh
-
-RUN /bootstrap/01-root-firstpass/002-install-python.sh
-
-RUN /bootstrap/01-root-firstpass/003-install-mariadb.sh
-
-RUN /bootstrap/01-root-firstpass/004-create-airflow-user.sh
-
-RUN /bootstrap/01-root-firstpass/005-install-aws-cli.sh
-
-RUN /bootstrap/01-root-firstpass/100-install-needed-dnf-packages.sh
-
-
-#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
-# END marker for root user, first pass bootstrapping steps.
-#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
-
-
-
-#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
-# > BEGINNING marker for airflow user bootstrapping steps.
-#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
-
-# Switch to 'airflow' user and update the PATH environment variable.
-USER airflow
-ENV PATH=${PATH_AIRFLOW_USER} 
-
-
-RUN /bootstrap/02-airflow/001-install-airflow.sh
-
-
-# Revert the PATH and user.
-ENV PATH=${PATH_DEFAULT}
-USER root
-
-#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
-# > END marker for airflow user bootstrapping steps.
-#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
-
-
+COPY ./bootstrap-dev /bootstrap-dev
+RUN chmod -R +x /bootstrap-dev
 
 #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
-# BEGINNING marker for root user, second pass bootstrapping steps.
+# BEGINNING marker for dev bootstrapping steps.
 #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
 
-# Those steps are also executed as the root user. However, they rely on the
-# successfull execution of the airflow user bootstrapping steps. For example,
-# giving ownership of the Airflow home user to the 'airflow' user requires the
-# the folder to be fully setup first.
-
-
-RUN /bootstrap/03-root-secondpass/001-create-mwaa-dir.sh
-
-RUN /bootstrap/03-root-secondpass/100-chown-airflow-folder.sh
-
-RUN /bootstrap/03-root-secondpass/200-devonly-install-dev-tools.sh
-
+RUN /bootstrap-dev/001-devonly-install-dev-tools.sh
 
 #<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
-# END marker for root user, second pass bootstrapping steps.
+# END marker for dev bootstrapping steps.
 #<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
 
 # remove bootstrapping files.
-RUN rm -rf /bootstrap
-
-# Create a volume for syncing files with the sidecar. The actual folder
-# is created by the `001-create-mwaa-dir.sh` script.
-VOLUME ["${MWAA_HOME}"]
-
-# TODO We should only expose this port if the command is 'webserver'.
-EXPOSE 8080
-
-ENV PATH=${PATH_AIRFLOW_USER}
-ENV PYTHONPATH="/python"
-
-WORKDIR ${AIRFLOW_USER_HOME}
-
-# Copy python files.
-COPY ./python /python
-
-
+RUN rm -rf /bootstrap-dev
 
 USER airflow