diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml index b4c95115..1d7d53a1 100644 --- a/.github/workflows/snapshot.yml +++ b/.github/workflows/snapshot.yml @@ -38,14 +38,14 @@ jobs: matrix: java_version: [8, 11] build: - - flink_version: 1.18-SNAPSHOT + - flink_version: 1.19-SNAPSHOT branch: dev-master + - flink_version: 1.18-SNAPSHOT + branch: dev-1.18 - flink_version: 1.17-SNAPSHOT branch: dev-1.17 - flink_version: 1.16-SNAPSHOT branch: dev-1.16 - - flink_version: 1.15-SNAPSHOT - branch: dev-1.15 steps: - uses: actions/checkout@v3 with: diff --git a/1.18/scala_2.12-java11-ubuntu/Dockerfile b/1.18/scala_2.12-java11-ubuntu/Dockerfile new file mode 100644 index 00000000..a53d9602 --- /dev/null +++ b/1.18/scala_2.12-java11-ubuntu/Dockerfile @@ -0,0 +1,94 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +FROM eclipse-temurin:11-jre-jammy + +# Install dependencies +RUN set -ex; \ + apt-get update; \ + apt-get -y install gpg libsnappy1v5 gettext-base libjemalloc-dev; \ + rm -rf /var/lib/apt/lists/* + +# Grab gosu for easy step-down from root +ENV GOSU_VERSION 1.11 +RUN set -ex; \ + wget -nv -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)"; \ + wget -nv -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + for server in ha.pool.sks-keyservers.net $(shuf -e \ + hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu) ; do \ + gpg --batch --keyserver "$server" --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && break || : ; \ + done && \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + chmod +x /usr/local/bin/gosu; \ + gosu nobody true + +# Configure Flink version +ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.18.0/flink-1.18.0-bin-scala_2.12.tgz \ + FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.18.0/flink-1.18.0-bin-scala_2.12.tgz.asc \ + GPG_KEY=96AE0E32CBE6E0753CE6DF6CB078D1D3253A8D82 \ + CHECK_GPG=true + +# Prepare environment +ENV FLINK_HOME=/opt/flink +ENV PATH=$FLINK_HOME/bin:$PATH +RUN groupadd --system --gid=9999 flink && \ + useradd --system --home-dir $FLINK_HOME --uid=9999 --gid=flink flink +WORKDIR $FLINK_HOME + +# Install Flink +RUN set -ex; \ + wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \ + \ + if [ "$CHECK_GPG" = "true" ]; then \ + wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \ + export GNUPGHOME="$(mktemp -d)"; \ + for server in ha.pool.sks-keyservers.net $(shuf -e \ + hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu) ; do \ + gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \ + done && \ + gpg --batch --verify flink.tgz.asc flink.tgz; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" flink.tgz.asc; \ + fi; \ + \ + tar -xf flink.tgz --strip-components=1; \ + rm flink.tgz; \ + \ + chown -R flink:flink .; \ + \ + # Replace default REST/RPC endpoint bind address to use the container's network interface \ + sed -i 's/rest.address: localhost/rest.address: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \ + sed -i 's/rest.bind-address: localhost/rest.bind-address: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \ + sed -i 's/jobmanager.bind-host: localhost/jobmanager.bind-host: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \ + sed -i 's/taskmanager.bind-host: localhost/taskmanager.bind-host: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \ + sed -i '/taskmanager.host: localhost/d' $FLINK_HOME/conf/flink-conf.yaml; + +# Configure container +COPY docker-entrypoint.sh / +ENTRYPOINT ["/docker-entrypoint.sh"] +EXPOSE 6123 8081 +CMD ["help"] diff --git a/1.18/scala_2.12-java11-ubuntu/docker-entrypoint.sh b/1.18/scala_2.12-java11-ubuntu/docker-entrypoint.sh new file mode 100755 index 00000000..8b0350e2 --- /dev/null +++ b/1.18/scala_2.12-java11-ubuntu/docker-entrypoint.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash + +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +COMMAND_STANDALONE="standalone-job" +COMMAND_HISTORY_SERVER="history-server" + +# If unspecified, the hostname of the container is taken as the JobManager address +JOB_MANAGER_RPC_ADDRESS=${JOB_MANAGER_RPC_ADDRESS:-$(hostname -f)} +CONF_FILE="${FLINK_HOME}/conf/flink-conf.yaml" + +drop_privs_cmd() { + if [ $(id -u) != 0 ]; then + # Don't need to drop privs if EUID != 0 + return + elif [ -x /sbin/su-exec ]; then + # Alpine + echo su-exec flink + else + # Others + echo gosu flink + fi +} + +copy_plugins_if_required() { + if [ -z "$ENABLE_BUILT_IN_PLUGINS" ]; then + return 0 + fi + + echo "Enabling required built-in plugins" + for target_plugin in $(echo "$ENABLE_BUILT_IN_PLUGINS" | tr ';' ' '); do + echo "Linking ${target_plugin} to plugin directory" + plugin_name=${target_plugin%.jar} + + mkdir -p "${FLINK_HOME}/plugins/${plugin_name}" + if [ ! -e "${FLINK_HOME}/opt/${target_plugin}" ]; then + echo "Plugin ${target_plugin} does not exist. Exiting." + exit 1 + else + ln -fs "${FLINK_HOME}/opt/${target_plugin}" "${FLINK_HOME}/plugins/${plugin_name}" + echo "Successfully enabled ${target_plugin}" + fi + done +} + +set_config_option() { + local option=$1 + local value=$2 + + # escape periods for usage in regular expressions + local escaped_option=$(echo ${option} | sed -e "s/\./\\\./g") + + # either override an existing entry, or append a new one + if grep -E "^${escaped_option}:.*" "${CONF_FILE}" > /dev/null; then + sed -i -e "s/${escaped_option}:.*/$option: $value/g" "${CONF_FILE}" + else + echo "${option}: ${value}" >> "${CONF_FILE}" + fi +} + +prepare_configuration() { + set_config_option jobmanager.rpc.address ${JOB_MANAGER_RPC_ADDRESS} + set_config_option blob.server.port 6124 + set_config_option query.server.port 6125 + + if [ -n "${TASK_MANAGER_NUMBER_OF_TASK_SLOTS}" ]; then + set_config_option taskmanager.numberOfTaskSlots ${TASK_MANAGER_NUMBER_OF_TASK_SLOTS} + fi + + if [ -n "${FLINK_PROPERTIES}" ]; then + echo "${FLINK_PROPERTIES}" >> "${CONF_FILE}" + fi + envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}" +} + +maybe_enable_jemalloc() { + if [ "${DISABLE_JEMALLOC:-false}" == "false" ]; then + JEMALLOC_PATH="/usr/lib/$(uname -m)-linux-gnu/libjemalloc.so" + JEMALLOC_FALLBACK="/usr/lib/x86_64-linux-gnu/libjemalloc.so" + if [ -f "$JEMALLOC_PATH" ]; then + export LD_PRELOAD=$LD_PRELOAD:$JEMALLOC_PATH + elif [ -f "$JEMALLOC_FALLBACK" ]; then + export LD_PRELOAD=$LD_PRELOAD:$JEMALLOC_FALLBACK + else + if [ "$JEMALLOC_PATH" = "$JEMALLOC_FALLBACK" ]; then + MSG_PATH=$JEMALLOC_PATH + else + MSG_PATH="$JEMALLOC_PATH and $JEMALLOC_FALLBACK" + fi + echo "WARNING: attempted to load jemalloc from $MSG_PATH but the library couldn't be found. glibc will be used instead." + fi + fi +} + +maybe_enable_jemalloc + +copy_plugins_if_required + +prepare_configuration + +args=("$@") +if [ "$1" = "help" ]; then + printf "Usage: $(basename "$0") (jobmanager|${COMMAND_STANDALONE}|taskmanager|${COMMAND_HISTORY_SERVER})\n" + printf " Or $(basename "$0") help\n\n" + printf "By default, Flink image adopts jemalloc as default memory allocator. This behavior can be disabled by setting the 'DISABLE_JEMALLOC' environment variable to 'true'.\n" + exit 0 +elif [ "$1" = "jobmanager" ]; then + args=("${args[@]:1}") + + echo "Starting Job Manager" + + exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground "${args[@]}" +elif [ "$1" = ${COMMAND_STANDALONE} ]; then + args=("${args[@]:1}") + + echo "Starting Job Manager" + + exec $(drop_privs_cmd) "$FLINK_HOME/bin/standalone-job.sh" start-foreground "${args[@]}" +elif [ "$1" = ${COMMAND_HISTORY_SERVER} ]; then + args=("${args[@]:1}") + + echo "Starting History Server" + + exec $(drop_privs_cmd) "$FLINK_HOME/bin/historyserver.sh" start-foreground "${args[@]}" +elif [ "$1" = "taskmanager" ]; then + args=("${args[@]:1}") + + echo "Starting Task Manager" + + exec $(drop_privs_cmd) "$FLINK_HOME/bin/taskmanager.sh" start-foreground "${args[@]}" +fi + +args=("${args[@]}") + +# Running command in pass-through mode +exec $(drop_privs_cmd) "${args[@]}" diff --git a/1.18/scala_2.12-java11-ubuntu/release.metadata b/1.18/scala_2.12-java11-ubuntu/release.metadata new file mode 100644 index 00000000..cd4f723c --- /dev/null +++ b/1.18/scala_2.12-java11-ubuntu/release.metadata @@ -0,0 +1,2 @@ +Tags: 1.18.0-scala_2.12-java11, 1.18-scala_2.12-java11, scala_2.12-java11, 1.18.0-scala_2.12, 1.18-scala_2.12, scala_2.12, 1.18.0-java11, 1.18-java11, java11, 1.18.0, 1.18, latest +Architectures: amd64,arm64v8 diff --git a/1.18/scala_2.12-java17-ubuntu/Dockerfile b/1.18/scala_2.12-java17-ubuntu/Dockerfile new file mode 100644 index 00000000..2a41daea --- /dev/null +++ b/1.18/scala_2.12-java17-ubuntu/Dockerfile @@ -0,0 +1,94 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +FROM eclipse-temurin:17-jre-jammy + +# Install dependencies +RUN set -ex; \ + apt-get update; \ + apt-get -y install gpg libsnappy1v5 gettext-base libjemalloc-dev; \ + rm -rf /var/lib/apt/lists/* + +# Grab gosu for easy step-down from root +ENV GOSU_VERSION 1.11 +RUN set -ex; \ + wget -nv -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)"; \ + wget -nv -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + for server in ha.pool.sks-keyservers.net $(shuf -e \ + hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu) ; do \ + gpg --batch --keyserver "$server" --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && break || : ; \ + done && \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + chmod +x /usr/local/bin/gosu; \ + gosu nobody true + +# Configure Flink version +ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.18.0/flink-1.18.0-bin-scala_2.12.tgz \ + FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.18.0/flink-1.18.0-bin-scala_2.12.tgz.asc \ + GPG_KEY=96AE0E32CBE6E0753CE6DF6CB078D1D3253A8D82 \ + CHECK_GPG=true + +# Prepare environment +ENV FLINK_HOME=/opt/flink +ENV PATH=$FLINK_HOME/bin:$PATH +RUN groupadd --system --gid=9999 flink && \ + useradd --system --home-dir $FLINK_HOME --uid=9999 --gid=flink flink +WORKDIR $FLINK_HOME + +# Install Flink +RUN set -ex; \ + wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \ + \ + if [ "$CHECK_GPG" = "true" ]; then \ + wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \ + export GNUPGHOME="$(mktemp -d)"; \ + for server in ha.pool.sks-keyservers.net $(shuf -e \ + hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu) ; do \ + gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \ + done && \ + gpg --batch --verify flink.tgz.asc flink.tgz; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" flink.tgz.asc; \ + fi; \ + \ + tar -xf flink.tgz --strip-components=1; \ + rm flink.tgz; \ + \ + chown -R flink:flink .; \ + \ + # Replace default REST/RPC endpoint bind address to use the container's network interface \ + sed -i 's/rest.address: localhost/rest.address: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \ + sed -i 's/rest.bind-address: localhost/rest.bind-address: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \ + sed -i 's/jobmanager.bind-host: localhost/jobmanager.bind-host: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \ + sed -i 's/taskmanager.bind-host: localhost/taskmanager.bind-host: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \ + sed -i '/taskmanager.host: localhost/d' $FLINK_HOME/conf/flink-conf.yaml; + +# Configure container +COPY docker-entrypoint.sh / +ENTRYPOINT ["/docker-entrypoint.sh"] +EXPOSE 6123 8081 +CMD ["help"] diff --git a/1.18/scala_2.12-java17-ubuntu/docker-entrypoint.sh b/1.18/scala_2.12-java17-ubuntu/docker-entrypoint.sh new file mode 100755 index 00000000..8b0350e2 --- /dev/null +++ b/1.18/scala_2.12-java17-ubuntu/docker-entrypoint.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash + +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +COMMAND_STANDALONE="standalone-job" +COMMAND_HISTORY_SERVER="history-server" + +# If unspecified, the hostname of the container is taken as the JobManager address +JOB_MANAGER_RPC_ADDRESS=${JOB_MANAGER_RPC_ADDRESS:-$(hostname -f)} +CONF_FILE="${FLINK_HOME}/conf/flink-conf.yaml" + +drop_privs_cmd() { + if [ $(id -u) != 0 ]; then + # Don't need to drop privs if EUID != 0 + return + elif [ -x /sbin/su-exec ]; then + # Alpine + echo su-exec flink + else + # Others + echo gosu flink + fi +} + +copy_plugins_if_required() { + if [ -z "$ENABLE_BUILT_IN_PLUGINS" ]; then + return 0 + fi + + echo "Enabling required built-in plugins" + for target_plugin in $(echo "$ENABLE_BUILT_IN_PLUGINS" | tr ';' ' '); do + echo "Linking ${target_plugin} to plugin directory" + plugin_name=${target_plugin%.jar} + + mkdir -p "${FLINK_HOME}/plugins/${plugin_name}" + if [ ! -e "${FLINK_HOME}/opt/${target_plugin}" ]; then + echo "Plugin ${target_plugin} does not exist. Exiting." + exit 1 + else + ln -fs "${FLINK_HOME}/opt/${target_plugin}" "${FLINK_HOME}/plugins/${plugin_name}" + echo "Successfully enabled ${target_plugin}" + fi + done +} + +set_config_option() { + local option=$1 + local value=$2 + + # escape periods for usage in regular expressions + local escaped_option=$(echo ${option} | sed -e "s/\./\\\./g") + + # either override an existing entry, or append a new one + if grep -E "^${escaped_option}:.*" "${CONF_FILE}" > /dev/null; then + sed -i -e "s/${escaped_option}:.*/$option: $value/g" "${CONF_FILE}" + else + echo "${option}: ${value}" >> "${CONF_FILE}" + fi +} + +prepare_configuration() { + set_config_option jobmanager.rpc.address ${JOB_MANAGER_RPC_ADDRESS} + set_config_option blob.server.port 6124 + set_config_option query.server.port 6125 + + if [ -n "${TASK_MANAGER_NUMBER_OF_TASK_SLOTS}" ]; then + set_config_option taskmanager.numberOfTaskSlots ${TASK_MANAGER_NUMBER_OF_TASK_SLOTS} + fi + + if [ -n "${FLINK_PROPERTIES}" ]; then + echo "${FLINK_PROPERTIES}" >> "${CONF_FILE}" + fi + envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}" +} + +maybe_enable_jemalloc() { + if [ "${DISABLE_JEMALLOC:-false}" == "false" ]; then + JEMALLOC_PATH="/usr/lib/$(uname -m)-linux-gnu/libjemalloc.so" + JEMALLOC_FALLBACK="/usr/lib/x86_64-linux-gnu/libjemalloc.so" + if [ -f "$JEMALLOC_PATH" ]; then + export LD_PRELOAD=$LD_PRELOAD:$JEMALLOC_PATH + elif [ -f "$JEMALLOC_FALLBACK" ]; then + export LD_PRELOAD=$LD_PRELOAD:$JEMALLOC_FALLBACK + else + if [ "$JEMALLOC_PATH" = "$JEMALLOC_FALLBACK" ]; then + MSG_PATH=$JEMALLOC_PATH + else + MSG_PATH="$JEMALLOC_PATH and $JEMALLOC_FALLBACK" + fi + echo "WARNING: attempted to load jemalloc from $MSG_PATH but the library couldn't be found. glibc will be used instead." + fi + fi +} + +maybe_enable_jemalloc + +copy_plugins_if_required + +prepare_configuration + +args=("$@") +if [ "$1" = "help" ]; then + printf "Usage: $(basename "$0") (jobmanager|${COMMAND_STANDALONE}|taskmanager|${COMMAND_HISTORY_SERVER})\n" + printf " Or $(basename "$0") help\n\n" + printf "By default, Flink image adopts jemalloc as default memory allocator. This behavior can be disabled by setting the 'DISABLE_JEMALLOC' environment variable to 'true'.\n" + exit 0 +elif [ "$1" = "jobmanager" ]; then + args=("${args[@]:1}") + + echo "Starting Job Manager" + + exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground "${args[@]}" +elif [ "$1" = ${COMMAND_STANDALONE} ]; then + args=("${args[@]:1}") + + echo "Starting Job Manager" + + exec $(drop_privs_cmd) "$FLINK_HOME/bin/standalone-job.sh" start-foreground "${args[@]}" +elif [ "$1" = ${COMMAND_HISTORY_SERVER} ]; then + args=("${args[@]:1}") + + echo "Starting History Server" + + exec $(drop_privs_cmd) "$FLINK_HOME/bin/historyserver.sh" start-foreground "${args[@]}" +elif [ "$1" = "taskmanager" ]; then + args=("${args[@]:1}") + + echo "Starting Task Manager" + + exec $(drop_privs_cmd) "$FLINK_HOME/bin/taskmanager.sh" start-foreground "${args[@]}" +fi + +args=("${args[@]}") + +# Running command in pass-through mode +exec $(drop_privs_cmd) "${args[@]}" diff --git a/1.18/scala_2.12-java17-ubuntu/release.metadata b/1.18/scala_2.12-java17-ubuntu/release.metadata new file mode 100644 index 00000000..2a19f9ff --- /dev/null +++ b/1.18/scala_2.12-java17-ubuntu/release.metadata @@ -0,0 +1,2 @@ +Tags: 1.18.0-scala_2.12-java17, 1.18-scala_2.12-java17, scala_2.12-java17, 1.18.0-java17, 1.18-java17, java17 +Architectures: amd64,arm64v8 diff --git a/1.18/scala_2.12-java8-ubuntu/Dockerfile b/1.18/scala_2.12-java8-ubuntu/Dockerfile new file mode 100644 index 00000000..72f72823 --- /dev/null +++ b/1.18/scala_2.12-java8-ubuntu/Dockerfile @@ -0,0 +1,94 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +FROM eclipse-temurin:8-jre-jammy + +# Install dependencies +RUN set -ex; \ + apt-get update; \ + apt-get -y install gpg libsnappy1v5 gettext-base libjemalloc-dev; \ + rm -rf /var/lib/apt/lists/* + +# Grab gosu for easy step-down from root +ENV GOSU_VERSION 1.11 +RUN set -ex; \ + wget -nv -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)"; \ + wget -nv -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + for server in ha.pool.sks-keyservers.net $(shuf -e \ + hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu) ; do \ + gpg --batch --keyserver "$server" --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && break || : ; \ + done && \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + chmod +x /usr/local/bin/gosu; \ + gosu nobody true + +# Configure Flink version +ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.18.0/flink-1.18.0-bin-scala_2.12.tgz \ + FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.18.0/flink-1.18.0-bin-scala_2.12.tgz.asc \ + GPG_KEY=96AE0E32CBE6E0753CE6DF6CB078D1D3253A8D82 \ + CHECK_GPG=true + +# Prepare environment +ENV FLINK_HOME=/opt/flink +ENV PATH=$FLINK_HOME/bin:$PATH +RUN groupadd --system --gid=9999 flink && \ + useradd --system --home-dir $FLINK_HOME --uid=9999 --gid=flink flink +WORKDIR $FLINK_HOME + +# Install Flink +RUN set -ex; \ + wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \ + \ + if [ "$CHECK_GPG" = "true" ]; then \ + wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \ + export GNUPGHOME="$(mktemp -d)"; \ + for server in ha.pool.sks-keyservers.net $(shuf -e \ + hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu) ; do \ + gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \ + done && \ + gpg --batch --verify flink.tgz.asc flink.tgz; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" flink.tgz.asc; \ + fi; \ + \ + tar -xf flink.tgz --strip-components=1; \ + rm flink.tgz; \ + \ + chown -R flink:flink .; \ + \ + # Replace default REST/RPC endpoint bind address to use the container's network interface \ + sed -i 's/rest.address: localhost/rest.address: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \ + sed -i 's/rest.bind-address: localhost/rest.bind-address: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \ + sed -i 's/jobmanager.bind-host: localhost/jobmanager.bind-host: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \ + sed -i 's/taskmanager.bind-host: localhost/taskmanager.bind-host: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \ + sed -i '/taskmanager.host: localhost/d' $FLINK_HOME/conf/flink-conf.yaml; + +# Configure container +COPY docker-entrypoint.sh / +ENTRYPOINT ["/docker-entrypoint.sh"] +EXPOSE 6123 8081 +CMD ["help"] diff --git a/1.18/scala_2.12-java8-ubuntu/docker-entrypoint.sh b/1.18/scala_2.12-java8-ubuntu/docker-entrypoint.sh new file mode 100755 index 00000000..8b0350e2 --- /dev/null +++ b/1.18/scala_2.12-java8-ubuntu/docker-entrypoint.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash + +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +COMMAND_STANDALONE="standalone-job" +COMMAND_HISTORY_SERVER="history-server" + +# If unspecified, the hostname of the container is taken as the JobManager address +JOB_MANAGER_RPC_ADDRESS=${JOB_MANAGER_RPC_ADDRESS:-$(hostname -f)} +CONF_FILE="${FLINK_HOME}/conf/flink-conf.yaml" + +drop_privs_cmd() { + if [ $(id -u) != 0 ]; then + # Don't need to drop privs if EUID != 0 + return + elif [ -x /sbin/su-exec ]; then + # Alpine + echo su-exec flink + else + # Others + echo gosu flink + fi +} + +copy_plugins_if_required() { + if [ -z "$ENABLE_BUILT_IN_PLUGINS" ]; then + return 0 + fi + + echo "Enabling required built-in plugins" + for target_plugin in $(echo "$ENABLE_BUILT_IN_PLUGINS" | tr ';' ' '); do + echo "Linking ${target_plugin} to plugin directory" + plugin_name=${target_plugin%.jar} + + mkdir -p "${FLINK_HOME}/plugins/${plugin_name}" + if [ ! -e "${FLINK_HOME}/opt/${target_plugin}" ]; then + echo "Plugin ${target_plugin} does not exist. Exiting." + exit 1 + else + ln -fs "${FLINK_HOME}/opt/${target_plugin}" "${FLINK_HOME}/plugins/${plugin_name}" + echo "Successfully enabled ${target_plugin}" + fi + done +} + +set_config_option() { + local option=$1 + local value=$2 + + # escape periods for usage in regular expressions + local escaped_option=$(echo ${option} | sed -e "s/\./\\\./g") + + # either override an existing entry, or append a new one + if grep -E "^${escaped_option}:.*" "${CONF_FILE}" > /dev/null; then + sed -i -e "s/${escaped_option}:.*/$option: $value/g" "${CONF_FILE}" + else + echo "${option}: ${value}" >> "${CONF_FILE}" + fi +} + +prepare_configuration() { + set_config_option jobmanager.rpc.address ${JOB_MANAGER_RPC_ADDRESS} + set_config_option blob.server.port 6124 + set_config_option query.server.port 6125 + + if [ -n "${TASK_MANAGER_NUMBER_OF_TASK_SLOTS}" ]; then + set_config_option taskmanager.numberOfTaskSlots ${TASK_MANAGER_NUMBER_OF_TASK_SLOTS} + fi + + if [ -n "${FLINK_PROPERTIES}" ]; then + echo "${FLINK_PROPERTIES}" >> "${CONF_FILE}" + fi + envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}" +} + +maybe_enable_jemalloc() { + if [ "${DISABLE_JEMALLOC:-false}" == "false" ]; then + JEMALLOC_PATH="/usr/lib/$(uname -m)-linux-gnu/libjemalloc.so" + JEMALLOC_FALLBACK="/usr/lib/x86_64-linux-gnu/libjemalloc.so" + if [ -f "$JEMALLOC_PATH" ]; then + export LD_PRELOAD=$LD_PRELOAD:$JEMALLOC_PATH + elif [ -f "$JEMALLOC_FALLBACK" ]; then + export LD_PRELOAD=$LD_PRELOAD:$JEMALLOC_FALLBACK + else + if [ "$JEMALLOC_PATH" = "$JEMALLOC_FALLBACK" ]; then + MSG_PATH=$JEMALLOC_PATH + else + MSG_PATH="$JEMALLOC_PATH and $JEMALLOC_FALLBACK" + fi + echo "WARNING: attempted to load jemalloc from $MSG_PATH but the library couldn't be found. glibc will be used instead." + fi + fi +} + +maybe_enable_jemalloc + +copy_plugins_if_required + +prepare_configuration + +args=("$@") +if [ "$1" = "help" ]; then + printf "Usage: $(basename "$0") (jobmanager|${COMMAND_STANDALONE}|taskmanager|${COMMAND_HISTORY_SERVER})\n" + printf " Or $(basename "$0") help\n\n" + printf "By default, Flink image adopts jemalloc as default memory allocator. This behavior can be disabled by setting the 'DISABLE_JEMALLOC' environment variable to 'true'.\n" + exit 0 +elif [ "$1" = "jobmanager" ]; then + args=("${args[@]:1}") + + echo "Starting Job Manager" + + exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground "${args[@]}" +elif [ "$1" = ${COMMAND_STANDALONE} ]; then + args=("${args[@]:1}") + + echo "Starting Job Manager" + + exec $(drop_privs_cmd) "$FLINK_HOME/bin/standalone-job.sh" start-foreground "${args[@]}" +elif [ "$1" = ${COMMAND_HISTORY_SERVER} ]; then + args=("${args[@]:1}") + + echo "Starting History Server" + + exec $(drop_privs_cmd) "$FLINK_HOME/bin/historyserver.sh" start-foreground "${args[@]}" +elif [ "$1" = "taskmanager" ]; then + args=("${args[@]:1}") + + echo "Starting Task Manager" + + exec $(drop_privs_cmd) "$FLINK_HOME/bin/taskmanager.sh" start-foreground "${args[@]}" +fi + +args=("${args[@]}") + +# Running command in pass-through mode +exec $(drop_privs_cmd) "${args[@]}" diff --git a/1.18/scala_2.12-java8-ubuntu/release.metadata b/1.18/scala_2.12-java8-ubuntu/release.metadata new file mode 100644 index 00000000..fccb4b07 --- /dev/null +++ b/1.18/scala_2.12-java8-ubuntu/release.metadata @@ -0,0 +1,2 @@ +Tags: 1.18.0-scala_2.12-java8, 1.18-scala_2.12-java8, scala_2.12-java8, 1.18.0-java8, 1.18-java8, java8 +Architectures: amd64,arm64v8 diff --git a/README.md b/README.md index ae52863f..ddf2cd33 100644 --- a/README.md +++ b/README.md @@ -121,10 +121,10 @@ Checklist for the `dev` branch: - [ ] The GPG key ID of the key used to sign the release has been added to `add-version.sh` and committed with the message `Add GPG key for x.y.z release` - [ ] `./add-version.sh -r x.y -f x.y.z` has been run on the respective dev branch -- [ ] `.github/workflows/snapshot.yml` in `master` has each supported version being mapped to the right branch. - Deprecated versions should have been removed. Checklist for the `master` branch: +- [ ] `.github/workflows/snapshot.yml` in `master` has each supported version being mapped to the right branch. + Deprecated versions should have been removed. - [ ] _(new major releases only)_ any unsupported Flink major version Dockerfiles have been removed (only two `x.y/` directories should be present) - [ ] _(new minor releases only)_ any existing generated files for the same major version have been