Merge branch 'master' into kangda-workspace

.bazelrc

@@ -2,7 +2,7 @@
 
 # Required by envoy and its tests
 build --define=grpc_no_ares=true
-build --cxxopt=-std=c++17 --host_cxxopt=-std=c++17
+build --cxxopt=-std=c++20 --host_cxxopt=-std=c++20
 
 # We already have absl in the build, define absl=1 to tell googletest to use absl for backtrace.
 build --define absl=1

Makefile

@@ -253,8 +253,13 @@ tools.buildifier:
 		go install github.com/bazelbuild/buildtools/buildifier@latest; \
 	fi
 
+# beautysh is already installed in the newer prow image with pipx
+# TODO(shuoyang2016): remove this after we switch prow to the
+# latest version.
 tools.beautysh:
-	@command -v beautysh  >/dev/null ; if [ $$? -ne 0 ]; then \
+	@command -v pipx  >/dev/null;\
+	PIPX_INSTALLED=$$?;\
+	command -v beautysh >/dev/null; if [ $$? -ne 0 ] && [ $${PIPX_INSTALLED} -ne 0 ]; then \
 		echo "--> installing beautysh"; \
 		pip3 install --user beautysh; \
 	fi
@@ -298,8 +303,13 @@ clang-format:
 	@echo $(CPP_PROTO_FILES) | xargs clang-format-14 -i
 
 shell-format: tools.beautysh
-	@echo "--> formatting shell scripts with 'beautysh' tool"
-	@git ls-files "*.sh" | xargs ${HOME}/.local/bin/beautysh -i 2
+	@command -v pipx  >/dev/null;\
+	PIPX_INSTALLED=$$?;\
+	echo "--> formatting shell scripts with 'beautysh' tool"; \
+	if [ $${PIPX_INSTALLED} -ne 0 ]; then \
+	  git ls-files "*.sh" | xargs ${HOME}/.local/bin/beautysh -i 2; \
+	else git ls-files "*.sh" | xargs pipx run beautysh -i 2; \
+        fi
 
 .PHONY: format.check
 format.check: tools.goimports

WORKSPACE

@@ -39,9 +39,9 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 # 3) Check if envoy_build_config/extensions_build_config.bzl is up-to-date.
 # Try to match it with the one in source/extensions and comment out unneeded extensions.
 
-ENVOY_SHA1 = "6b9db09c69965d5bfb37bdd29693f8b7f9e9e9ec"  # v1.27.1, 2023.10.11
+ENVOY_SHA1 = "816188b86a0a52095b116b107f576324082c7c02"  # v1.30.1
 
-ENVOY_SHA256 = "d6cde20343d67fa4e25b9047bd805c522ece80b8058f1f311cb90ee7f3287f63"
+ENVOY_SHA256 = "41064ee8fbafc2ac3fd0d6531a106139adbbea3585defff22fdb99e06d7862e5"
 
 http_archive(
     name = "envoy",
@@ -105,9 +105,10 @@ pip_install(
     requirements = "@com_github_grpc_grpc//:requirements.bazel.txt",
 )
 
+load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps", "grpc_test_only_deps")
+
 # ==============================================================================
 load("@com_github_grpc_grpc//bazel:grpc_python_deps.bzl", "grpc_python_deps")
-load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps", "grpc_test_only_deps")
 
 grpc_python_deps()
 

docker/Dockerfile-prow-env

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM debian:buster
+FROM debian:bullseye
 LABEL maintainer="[email protected]"
 
 # add env we can debug with the image name:tag
@@ -22,13 +22,13 @@ ENV IMAGE=${IMAGE_ARG}
 
 RUN apt-get update -y
 RUN apt-get -y install \
-    wget make cmake python3 python3-pip pkg-config coreutils \
+    wget make cmake python3 python3-pip python3-venv pkg-config coreutils \
     zlib1g-dev curl libtool automake zip time rsync ninja-build \
     git bash-completion jq default-jdk python3-distutils libicu-dev libbrotli-dev
 
 
 # install nodejs, which is needed for integration tests
-RUN sh -c 'curl -sL https://deb.nodesource.com/setup_12.x | bash -'
+RUN sh -c 'curl -sL https://deb.nodesource.com/setup_20.x | bash -'
 RUN apt-get install -y nodejs
 
 # install Bazelisk
@@ -38,7 +38,7 @@ RUN wget -O /usr/local/bin/bazelisk https://github.com/bazelbuild/bazelisk/relea
 # install clang-14 and associated tools (new envoy)
 # see https://apt.llvm.org/ for exhaustive list of all llvm related packages
 RUN wget -O- https://apt.llvm.org/llvm-snapshot.gpg.key| apt-key add - && \
-    echo "deb https://apt.llvm.org/buster/ llvm-toolchain-buster-14 main" >> /etc/apt/sources.list && \
+    echo "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-14 main" >> /etc/apt/sources.list && \
     apt-get update && \
     apt-get install -y llvm-14 llvm-14-dev libclang-14-dev clang-14 \
         lld-14 clang-tools-14 clang-format-14 libc++-dev xz-utils libclang-rt-14-dev
@@ -59,6 +59,11 @@ RUN wget -q "https://go.dev/dl/${GO_TARBALL}" && \
 # Install buildifier
 RUN go install github.com/bazelbuild/buildtools/buildifier@latest
 
+RUN pip install pipx
+
+# Install beautysh
+RUN pipx install beautysh
+
 # install gcloud package
 RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz
 RUN mkdir -p /usr/local/gcloud \

docker/Dockerfile-proxy.tmpl

@@ -18,10 +18,7 @@ FROM _ENVOY_IMAGE_SHA_NAME
 RUN apk add --update --no-cache openssl
 
 # Install python3
-ENV PYTHONUNBUFFERED=1
-RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python
-RUN python3 -m ensurepip
-RUN pip3 install --no-cache --upgrade pip setuptools
+RUN apk add --no-cache python3 py3-pip
 
 ENV PATH /bin:$PATH
 

prow/e2e-gcloud-build-image.sh

@@ -73,28 +73,28 @@ echo "=== Test 1: Specify a fully qualified version. ==="
 EXPECTED_IMAGE_NAME=$(formImageName "2.7.0")
 cleanupOldImage "${EXPECTED_IMAGE_NAME}"
 ${ROOT}/docker/serverless/gcloud_build_image \
-    -s "${SERVICE_NAME}" \
-    -c "${CONFIG_ID}" \
-    -p "${PROJECT_NAME}" \
-    -v "2.7.0"
+  -s "${SERVICE_NAME}" \
+  -c "${CONFIG_ID}" \
+  -p "${PROJECT_NAME}" \
+  -v "2.7.0"
 expectImage "${EXPECTED_IMAGE_NAME}"
 
 echo "=== Test 2: Specify a minor version. ==="
 EXPECTED_IMAGE_NAME=$(formImageName "2.4.0")
 cleanupOldImage "${EXPECTED_IMAGE_NAME}"
 ${ROOT}/docker/serverless/gcloud_build_image \
-    -s "${SERVICE_NAME}" \
-    -c "${CONFIG_ID}" \
-    -p "${PROJECT_NAME}" \
-    -v "2.4"
+  -s "${SERVICE_NAME}" \
+  -c "${CONFIG_ID}" \
+  -p "${PROJECT_NAME}" \
+  -v "2.4"
 expectImage "${EXPECTED_IMAGE_NAME}"
 
 echo "=== Test 3: Sepcify an invalid version fails. ==="
 if ${ROOT}/docker/serverless/gcloud_build_image \
-    -s "${SERVICE_NAME}" \
-    -c "${CONFIG_ID}" \
-    -p "${PROJECT_NAME}" \
-    -v "2.11.47"; then
+  -s "${SERVICE_NAME}" \
+  -c "${CONFIG_ID}" \
+  -p "${PROJECT_NAME}" \
+  -v "2.11.47"; then
   error_exit "Script should fail for invalid version."
 else
   echo "Script failed as expected."
@@ -104,21 +104,21 @@ echo "=== Test 4: Specify a custom image. ==="
 EXPECTED_IMAGE_NAME=$(formImageName "custom")
 cleanupOldImage "${EXPECTED_IMAGE_NAME}"
 ${ROOT}/docker/serverless/gcloud_build_image \
-    -s "${SERVICE_NAME}" \
-    -c "${CONFIG_ID}" \
-    -p "${PROJECT_NAME}" \
-    -i "gcr.io/cloudesf-testing/apiproxy-serverless:gcloud-build-image-test"
+  -s "${SERVICE_NAME}" \
+  -c "${CONFIG_ID}" \
+  -p "${PROJECT_NAME}" \
+  -i "gcr.io/cloudesf-testing/apiproxy-serverless:gcloud-build-image-test"
 expectImage "${EXPECTED_IMAGE_NAME}"
 
 echo "=== Test 5: Specify a GAR_REPOSITORY_IMAGE_PREFIX with -g flag. ==="
 EXPECTED_IMAGE_NAME=$(formGarImageName "2.30.3")
 cleanupOldImage "${EXPECTED_IMAGE_NAME}"
 ${ROOT}/docker/serverless/gcloud_build_image \
-    -s "${SERVICE_NAME}" \
-    -c "${CONFIG_ID}" \
-    -p "${PROJECT_NAME}" \
-    -v "2.30.3" \
-    -g "${GAR_REPOSITORY_IMAGE_PREFIX}"
+  -s "${SERVICE_NAME}" \
+  -c "${CONFIG_ID}" \
+  -p "${PROJECT_NAME}" \
+  -v "2.30.3" \
+  -g "${GAR_REPOSITORY_IMAGE_PREFIX}"
 expectImage "${EXPECTED_IMAGE_NAME}"
 
 echo "=== Test 6: When no ESP version is specified, the script uses the latest ESPv2 release. ==="
@@ -127,7 +127,7 @@ echo "=== Test 6: When no ESP version is specified, the script uses the latest E
 # That means we don't have a reliable way of checking if the output is correct.
 # So just test the script passes, and allow the developer to manually verify the output.
 ${ROOT}/docker/serverless/gcloud_build_image \
-    -s "${SERVICE_NAME}" \
-    -c "${CONFIG_ID}" \
-    -p "${PROJECT_NAME}"
+  -s "${SERVICE_NAME}" \
+  -c "${CONFIG_ID}" \
+  -p "${PROJECT_NAME}"
 echo ">>> WARNING: For the test above, manually verify the output version of the image is expected."

prow/gcpproxy-e2e.sh

@@ -36,7 +36,7 @@ function runE2E() {
       g) local backend="${OPTARG}" ;;
       m) local apiproxy_image="${OPTARG}" ;;
       R) local rollout_strategy="${OPTARG}" ;;
-      S) local using_sa_cred='true';;
+      S) local using_sa_cred='true' ;;
       t) local test_type="$(echo ${OPTARG} | tr '[A-Z]' '[a-z]')" ;;
     esac
   done
@@ -89,7 +89,7 @@ case ${TEST_CASE} in
   "tight-http-bookstore-managed")
     runE2E -p "gke" -c "tight" -t "http" -g "bookstore" -R "managed" -m "$(get_proxy_image_name_with_sha)"
     ;;
-    "tight-http-bookstore-managed-using-sa-cred")
+  "tight-http-bookstore-managed-using-sa-cred")
     runE2E -p "gke" -c "tight" -t "http" -g "bookstore" -R "managed" -S -m "$(get_proxy_image_name_with_sha)"
     ;;
   "tight-grpc-echo-managed")
@@ -107,7 +107,7 @@ case ${TEST_CASE} in
   "cloud-run-cloud-function-http-bookstore")
     runE2E -p "cloud-run" -f "cloud-function" -t "http" -g "bookstore" -R "managed" -m "$(get_serverless_image_name_with_sha)"
     ;;
-    "cloud-run-app-engine-http-bookstore")
+  "cloud-run-app-engine-http-bookstore")
     runE2E -p "cloud-run" -f "app-engine" -t "http" -g "bookstore" -R "managed" -m "$(get_serverless_image_name_with_sha)"
     ;;
   "anthos-cloud-run-anthos-cloud-run-http-bookstore")

prow/gcpproxy-presubmit.sh

@@ -63,7 +63,7 @@ make depend.install
 
 # GOOGLE_APPLICATION_CREDENTIALS will be set in our test environment but this env
 # var if set will be used by start_proxy.py as service_json_path.
- (unset GOOGLE_APPLICATION_CREDENTIALS; make test)
+(unset GOOGLE_APPLICATION_CREDENTIALS; make test)
 
 # c++ test
 echo '======================================================'

prow/janitor.sh

@@ -71,8 +71,8 @@ for PROJECT in ${PROJECT_IDS[@]}; do
 
     ### App Engines ###
     APP_ENGINES=$(gcloud app services list \
-    --filter="SERVICE ~ ^e2e-test-" \
-    --format="value(SERVICE)")
+        --filter="SERVICE ~ ^e2e-test-" \
+      --format="value(SERVICE)")
     for ap in ${APP_ENGINES} ; do
       echo "Deleting App Engine: ${ap}"
       gcloud app services delete "${ap}" \
@@ -86,7 +86,7 @@ for PROJECT in ${PROJECT_IDS[@]}; do
     FIREWALL_RULES=$(gcloud compute firewall-rules list \
         --filter="targetTags:(gke-e2e-cloud-run) \
         AND creationTimestamp < ${LIMIT_DATE}" \
-        --format="value(name)")
+      --format="value(name)")
 
     for rule in $FIREWALL_RULES ; do
       echo "Deleting Firewall rule: ${rule}"
@@ -101,19 +101,19 @@ for PROJECT in ${PROJECT_IDS[@]}; do
 
     TARGET_POOLS=$(gcloud compute target-pools list \
         --regions="${REGION}" \
-        --format='value(name)')
+      --format='value(name)')
 
     for targetpool in $TARGET_POOLS; do
       echo "Query Forwarding Rule for target pool ${targetpool}"
       forwardingitem=$(gcloud compute forwarding-rules list \
-        --filter=TARGET="https://www.googleapis.com/compute/v1/projects/$PROJECT/regions/$REGION/targetPools/$targetpool" \
+          --filter=TARGET="https://www.googleapis.com/compute/v1/projects/$PROJECT/regions/$REGION/targetPools/$targetpool" \
         --format='value(name)')
-        if [[ -z "$forwardingitem" ]]; then
-          echo "Deleting unused target pool ${targetpool}"
-          gcloud compute target-pools delete "${targetpool}" \
-            --region="${REGION}" \
-            --quiet
-        fi
+      if [[ -z "$forwardingitem" ]]; then
+        echo "Deleting unused target pool ${targetpool}"
+        gcloud compute target-pools delete "${targetpool}" \
+          --region="${REGION}" \
+          --quiet
+      fi
     done
     echo "Done cleaning up target pools without forwarding rules"
 
@@ -124,12 +124,12 @@ for PROJECT in ${PROJECT_IDS[@]}; do
     TARGET_POOLS=$(gcloud compute target-pools list \
         --regions="${REGION}" \
         --filter="creationTimestamp < ${LIMIT_DATE}" \
-        --format='value(name)')
+      --format='value(name)')
 
     for targetpool in $TARGET_POOLS; do
       echo "Detected cloud run target pool ${targetpool}, querying forwarding rule"
       forwardingitem=$(gcloud compute forwarding-rules list \
-        --filter=TARGET="https://www.googleapis.com/compute/v1/projects/$PROJECT/regions/$REGION/targetPools/$targetpool" \
+          --filter=TARGET="https://www.googleapis.com/compute/v1/projects/$PROJECT/regions/$REGION/targetPools/$targetpool" \
         --format='value(name)')
       echo "Deleting forwarding rule ${forwardingitem}"
       gcloud compute forwarding-rules delete "${forwardingitem}" \
@@ -149,8 +149,8 @@ for PROJECT in ${PROJECT_IDS[@]}; do
     ### Static IPs ###
     # Clean up static IPs that are reserved 1 day ago but not in use.
     STATIC_IPS=$(gcloud compute addresses list \
-      --filter="status=RESERVED AND creationTimestamp < ${LIMIT_DATE}" \
-      --regions="${REGION}" \
+        --filter="status=RESERVED AND creationTimestamp < ${LIMIT_DATE}" \
+        --regions="${REGION}" \
       --format="value(name)")
     for static_ip in $STATIC_IPS; do
       gcloud compute addresses delete ${static_ip} --region $REGION --quiet

scripts/format-examples.sh

@@ -23,12 +23,12 @@ shopt -s globstar
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 
 for filename in $ROOT/examples/**/*.json; do
-    echo "Formatting $filename"
-    TEMP_FILE=$(mktemp)
+  echo "Formatting $filename"
+  TEMP_FILE=$(mktemp)
 
-    # jq is a common bash utility used to format/sort/filter json.
-    # Sort keys (-S) for all fields (.) in the input file and output to the temp file.
-    jq -S '.' "$filename" > "$TEMP_FILE"
-    cp -f "$TEMP_FILE" "$filename"
-    rm "$TEMP_FILE"
+  # jq is a common bash utility used to format/sort/filter json.
+  # Sort keys (-S) for all fields (.) in the input file and output to the temp file.
+  jq -S '.' "$filename" > "$TEMP_FILE"
+  cp -f "$TEMP_FILE" "$filename"
+  rm "$TEMP_FILE"
 done

scripts/release/release-changelog.sh

@@ -50,11 +50,11 @@ DIRECTORY="."
 
 while getopts :s:l:d:n: arg; do
   case ${arg} in
-    s) SHA="${OPTARG}";;
-    l) LAST_COMMIT="${OPTARG}";;
-    d) DIRECTORY="${OPTARG}";;
-    n) VERSION="${OPTARG}";;
-    *) usage "Invalid option: -${OPTARG}";;
+    s) SHA="${OPTARG}" ;;
+    l) LAST_COMMIT="${OPTARG}" ;;
+    d) DIRECTORY="${OPTARG}" ;;
+    n) VERSION="${OPTARG}" ;;
+    *) usage "Invalid option: -${OPTARG}" ;;
   esac
 done
 
@@ -113,7 +113,7 @@ EOF
 echo $(pwd)
 git log ${LAST_COMMIT}..${SHA} --pretty="- %s%w(76,2)" \
   | perl -pe'BEGIN {undef $/;} s/\s+Change-Id:[^\n]*\s+/\n/gs;' \
-    >> "${CHANGELOG}"
+  >> "${CHANGELOG}"
 
 cat <<EOF >> "${CHANGELOG}"
 

scripts/release/release-new-branch.sh

@@ -38,8 +38,8 @@ SHA=''
 
 while getopts :n:s: arg; do
   case ${arg} in
-    s) SHA="${OPTARG}";;
-    *) usage "Invalid option: -${OPTARG}";;
+    s) SHA="${OPTARG}" ;;
+    *) usage "Invalid option: -${OPTARG}" ;;
   esac
 done