Add local CI scripts & rebase docker image (#394)

* Docker image uses `rapidsai/ci-conda` as the base per rapidsai/ops#2806
* Adds the ability to CI locally via `./ci/scripts/run_ci_local.sh build` & `./ci/scripts/run_ci_local.sh test`
* Fix type-o in benchmark stage causing benchmarks to be executed twice & preventing results from being published.
* Add `.tmp` and `.clangd` to `.gitignore`
* Document need to set `DOCKER_BUILDKIT=1` for docker builds.


fixes #393
fixes #397

Authors:
  - David Gardner (https://github.com/dagardner-nv)

Approvers:
  - Christopher Harris (https://github.com/cwharris)

URL: #394

.github/workflows/ci_pipe.yml

@@ -294,7 +294,7 @@ jobs:
         run: ./mrc/ci/scripts/github/benchmark.sh
       - name: post_benchmark
         shell: bash
-        run: ./mrc/ci/scripts/github/benchmark.sh
+        run: ./mrc/ci/scripts/github/post_benchmark.sh
 
 
   package:

.github/workflows/pull_request.yml

@@ -71,9 +71,9 @@ jobs:
       # Update conda package only for non PR branches. Use 'main' for main branch and 'dev' for all other branches
       conda_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }}
       # Build container
-      container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-230711
+      container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-230920
       # Test container
-      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-230711
+      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-230920
       # Info about the PR. Empty for non PR branches. Useful for extracting PR number, title, etc.
       pr_info: ${{ needs.prepare.outputs.pr_info }}
     secrets:

.gitignore

@@ -1,4 +1,5 @@
 /build*/
+.tmp
 *.engine
 .Dockerfile
 .gitignore
@@ -17,6 +18,9 @@ include/mrc/version.hpp
 .vscode/settings.json
 .vscode/tasks.json
 
+# Ignore user-defined clangd settings
+.clangd
+
 # Created by https://www.gitignore.io/api/vim,c++,cmake,python,synology
 
 ### C++ ###

README.md

@@ -151,7 +151,7 @@ pytest $MRC_ROOT/python
 ### Docker Installation
 A Dockerfile is provided at `$MRC_ROOT` and can be built with
 ```bash
-docker build -t mrc:latest .
+DOCKER_BUILDKIT=1 docker build -t mrc:latest .
 ```
 To run the container
 ```bash

ci/scripts/bootstrap_local_ci.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export WORKSPACE_TMP="$(pwd)/.tmp/local_ci_workspace"
+mkdir -p ${WORKSPACE_TMP}
+git clone ${GIT_URL} mrc
+cd mrc/
+git checkout ${GIT_BRANCH}
+git pull
+git checkout ${GIT_COMMIT}
+
+export MRC_ROOT=$(pwd)
+export WORKSPACE=${MRC_ROOT}
+export LOCAL_CI=1
+GH_SCRIPT_DIR="${MRC_ROOT}/ci/scripts/github"
+
+unset CMAKE_CUDA_COMPILER_LAUNCHER
+unset CMAKE_CXX_COMPILER_LAUNCHER
+unset CMAKE_C_COMPILER_LAUNCHER
+
+if [[ "${STAGE}" != "bash" ]]; then
+    # benchmark & codecov are composite stages, the rest are composed of a single shell script
+    if [[ "${STAGE}" == "benchmark" || "${STAGE}" == "codecov" ]]; then
+        CI_SCRIPT="${WORKSPACE_TMP}/ci_script.sh"
+        echo "#!/bin/bash" > ${CI_SCRIPT}
+        if [[ "${STAGE}" == "benchmark" ]]; then
+            echo "${GH_SCRIPT_DIR}/pre_benchmark.sh" >> ${CI_SCRIPT}
+            echo "${GH_SCRIPT_DIR}/benchmark.sh" >> ${CI_SCRIPT}
+            echo "${GH_SCRIPT_DIR}/post_benchmark.sh" >> ${CI_SCRIPT}
+        else
+            echo "${GH_SCRIPT_DIR}/build.sh" >> ${CI_SCRIPT}
+            echo "${GH_SCRIPT_DIR}/test_codecov.sh" >> ${CI_SCRIPT}
+        fi
+
+        chmod +x ${CI_SCRIPT}
+    else
+        if [[ "${STAGE}" =~ "build" ]]; then
+            CI_SCRIPT="${GH_SCRIPT_DIR}/build.sh"
+        elif [[ "${STAGE}" =~ "test" ]]; then
+            CI_SCRIPT="${GH_SCRIPT_DIR}/test.sh"
+        else
+            CI_SCRIPT="${GH_SCRIPT_DIR}/${STAGE}.sh"
+        fi
+    fi
+
+    ${CI_SCRIPT}
+fi

ci/scripts/github/build.sh

@@ -20,7 +20,12 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 
 update_conda_env
 
-CMAKE_CACHE_FLAGS="-DCCACHE_PROGRAM_PATH=$(which sccache) -DMRC_USE_CCACHE=ON"
+if [[ "${LOCAL_CI}" == "" ]]; then
+    CMAKE_CACHE_FLAGS="-DCCACHE_PROGRAM_PATH=$(which sccache) -DMRC_USE_CCACHE=ON"
+else
+    CMAKE_CACHE_FLAGS=""
+fi
+
 
 rapids-logger "Check versions"
 python3 --version
@@ -56,18 +61,20 @@ cmake -B build -G Ninja ${CMAKE_FLAGS} .
 rapids-logger "Building MRC"
 cmake --build build --parallel ${PARALLEL_LEVEL}
 
-rapids-logger "sccache usage for MRC build:"
-sccache --show-stats
+if [[ "${LOCAL_CI}" == "" ]]; then
+    rapids-logger "sccache usage for MRC build:"
+    sccache --show-stats
+fi
 
-if [[ "${BUILD_CC}" != "gcc-coverage" ]]; then
+if [[ "${BUILD_CC}" != "gcc-coverage" || ${LOCAL_CI} == "1" ]]; then
     rapids-logger "Archiving results"
     tar cfj "${WORKSPACE_TMP}/dot_cache.tar.bz" .cache
     tar cfj "${WORKSPACE_TMP}/build.tar.bz" build
     ls -lh ${WORKSPACE_TMP}/
 
     rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/"
-    aws s3 cp --no-progress "${WORKSPACE_TMP}/build.tar.bz" "${ARTIFACT_URL}/build.tar.bz"
-    aws s3 cp --no-progress "${WORKSPACE_TMP}/dot_cache.tar.bz" "${ARTIFACT_URL}/dot_cache.tar.bz"
+    upload_artifact "${WORKSPACE_TMP}/build.tar.bz"
+    upload_artifact "${WORKSPACE_TMP}/dot_cache.tar.bz"
 fi
 
 rapids-logger "Success"

ci/scripts/github/common.sh

@@ -56,7 +56,12 @@ export S3_URL="s3://rapids-downloads/ci/mrc"
 export DISPLAY_URL="https://downloads.rapids.ai/ci/mrc"
 export ARTIFACT_ENDPOINT="/pull-request/${PR_NUM}/${GIT_COMMIT}/${NVARCH}/${BUILD_CC}"
 export ARTIFACT_URL="${S3_URL}${ARTIFACT_ENDPOINT}"
-export DISPLAY_ARTIFACT_URL="${DISPLAY_URL}${ARTIFACT_ENDPOINT}"
+
+if [[ "${LOCAL_CI}" == "1" ]]; then
+    export DISPLAY_ARTIFACT_URL="${LOCAL_CI_TMP}"
+else
+    export DISPLAY_ARTIFACT_URL="${DISPLAY_URL}${ARTIFACT_ENDPOINT}"
+fi
 
 # Set sccache env vars
 export SCCACHE_S3_KEY_PREFIX=mrc-${NVARCH}-${BUILD_CC}
@@ -78,9 +83,11 @@ function update_conda_env() {
     # Deactivate the environment first before updating
     conda deactivate
 
-    # Make sure we have the conda-merge package installed
-    if [[ -z "$(conda list | grep conda-merge)" ]]; then
-        rapids-mamba-retry install -q -n mrc -c conda-forge "conda-merge>=0.2"
+    if [[ "${SKIP_CONDA_ENV_UPDATE}" == "" ]]; then
+        # Make sure we have the conda-merge package installed
+        if [[ -z "$(conda list | grep conda-merge)" ]]; then
+            rapids-mamba-retry install -q -n mrc -c conda-forge "conda-merge>=0.2"
+        fi
     fi
 
     # Create a temp directory which we store the combined environment file in
@@ -90,8 +97,10 @@ function update_conda_env() {
     # will clobber the last env update
     conda run -n mrc --live-stream conda-merge ${CONDA_ENV_YML} ${CONDA_CLANG_ENV_YML} ${CONDA_CI_ENV_YML} > ${condatmpdir}/merged_env.yml
 
-    # Update the conda env with prune remove excess packages (in case one was removed from the env)
-    rapids-mamba-retry env update -n mrc --prune --file ${condatmpdir}/merged_env.yml
+    if [[ "${SKIP_CONDA_ENV_UPDATE}" == "" ]]; then
+        # Update the conda env with prune remove excess packages (in case one was removed from the env)
+        rapids-mamba-retry env update -n mrc --prune --file ${condatmpdir}/merged_env.yml
+    fi
 
     # Delete the temp directory
     rm -rf ${condatmpdir}
@@ -105,7 +114,12 @@ function update_conda_env() {
 
 print_env_vars
 
-function fetch_base_branch() {
+function fetch_base_branch_gh_api() {
+    # For PRs, $GIT_BRANCH is like: pull-request/989
+    REPO_NAME=$(basename "${GITHUB_REPOSITORY}")
+    ORG_NAME="${GITHUB_REPOSITORY_OWNER}"
+    PR_NUM="${GITHUB_REF_NAME##*/}"
+
     rapids-logger "Retrieving base branch from GitHub API"
     [[ -n "$GH_TOKEN" ]] && CURL_HEADERS=('-H' "Authorization: token ${GH_TOKEN}")
     RESP=$(
@@ -115,25 +129,31 @@ function fetch_base_branch() {
         "${GITHUB_API_URL}/repos/${ORG_NAME}/${REPO_NAME}/pulls/${PR_NUM}"
     )
 
-    BASE_BRANCH=$(echo "${RESP}" | jq -r '.base.ref')
+    export BASE_BRANCH=$(echo "${RESP}" | jq -r '.base.ref')
 
     # Change target is the branch name we are merging into but due to the weird way jenkins does
     # the checkout it isn't recognized by git without the origin/ prefix
     export CHANGE_TARGET="origin/${BASE_BRANCH}"
-    git submodule update --init --recursive
-    rapids-logger "Base branch: ${BASE_BRANCH}"
 }
 
-function fetch_s3() {
-    ENDPOINT=$1
-    DESTINATION=$2
-    if [[ "${USE_S3_CURL}" == "1" ]]; then
-        curl -f "${DISPLAY_URL}${ENDPOINT}" -o "${DESTINATION}"
-        FETCH_STATUS=$?
+function fetch_base_branch_local() {
+    rapids-logger "Retrieving base branch from git"
+    git remote add upstream ${GIT_UPSTREAM_URL}
+    git fetch upstream --tags
+    source ${MRC_ROOT}/ci/scripts/common.sh
+    export BASE_BRANCH=$(get_base_branch)
+    export CHANGE_TARGET="upstream/${BASE_BRANCH}"
+}
+
+function fetch_base_branch() {
+    if [[ "${LOCAL_CI}" == "1" ]]; then
+        fetch_base_branch_local
     else
-        aws s3 cp --no-progress "${S3_URL}${ENDPOINT}" "${DESTINATION}"
-        FETCH_STATUS=$?
+        fetch_base_branch_gh_api
     fi
+
+    git submodule update --init --recursive
+    rapids-logger "Base branch: ${BASE_BRANCH}"
 }
 
 function show_conda_info() {
@@ -143,3 +163,25 @@ function show_conda_info() {
     conda config --show-sources
     conda list --show-channel-urls
 }
+
+function upload_artifact() {
+    FILE_NAME=$1
+    BASE_NAME=$(basename "${FILE_NAME}")
+    rapids-logger "Uploading artifact: ${BASE_NAME}"
+    if [[ "${LOCAL_CI}" == "1" ]]; then
+        cp ${FILE_NAME} "${LOCAL_CI_TMP}/${BASE_NAME}"
+    else
+        aws s3 cp --only-show-errors "${FILE_NAME}" "${ARTIFACT_URL}/${BASE_NAME}"
+        echo "- ${DISPLAY_ARTIFACT_URL}/${BASE_NAME}" >> ${GITHUB_STEP_SUMMARY}
+    fi
+}
+
+function download_artifact() {
+    ARTIFACT=$1
+    rapids-logger "Downloading ${ARTIFACT} from ${DISPLAY_ARTIFACT_URL}"
+    if [[ "${LOCAL_CI}" == "1" ]]; then
+        cp "${LOCAL_CI_TMP}/${ARTIFACT}" "${WORKSPACE_TMP}/${ARTIFACT}"
+    else
+        aws s3 cp --no-progress "${ARTIFACT_URL}/${ARTIFACT}" "${WORKSPACE_TMP}/${ARTIFACT}"
+    fi
+}

ci/scripts/github/conda.sh

@@ -16,6 +16,7 @@
 
 set -e
 
+CI_SCRIPT_ARGS="$@"
 source ${WORKSPACE}/ci/scripts/github/common.sh
 
 # Its important that we are in the base environment for the build
@@ -39,4 +40,15 @@ conda info
 rapids-logger "Building Conda Package"
 
 # Run the conda build and upload
-${MRC_ROOT}/ci/conda/recipes/run_conda_build.sh "$@"
+${MRC_ROOT}/ci/conda/recipes/run_conda_build.sh "${CI_SCRIPT_ARGS}"
+
+if [[ " ${CI_SCRIPT_ARGS} " =~ " upload " ]]; then
+   rapids-logger "Building Conda Package... Done"
+else
+   # if we didn't receive the upload argument, we can still upload the artifact to S3
+   tar cfj "${WORKSPACE_TMP}/conda.tar.bz" "${RAPIDS_CONDA_BLD_OUTPUT_DIR}"
+   ls -lh ${WORKSPACE_TMP}/
+
+   rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/"
+   upload_artifact "${WORKSPACE_TMP}/conda.tar.bz"
+fi

ci/scripts/github/docs.sh

@@ -39,6 +39,6 @@ rapids-logger "Tarring the docs"
 tar cfj "${WORKSPACE_TMP}/docs.tar.bz" build/docs/html
 
 rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/"
-aws s3 cp --no-progress "${WORKSPACE_TMP}/docs.tar.bz" "${ARTIFACT_URL}/docs.tar.bz"
+upload_artifact "${WORKSPACE_TMP}/docs.tar.bz"
 
 rapids-logger "Success"

ci/scripts/github/post_benchmark.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,6 +25,6 @@ cd $(dirname ${REPORTS_DIR})
 tar cfj ${WORKSPACE_TMP}/benchmark_reports.tar.bz $(basename ${REPORTS_DIR})
 
 rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/"
-aws s3 cp ${WORKSPACE_TMP}/benchmark_reports.tar.bz "${ARTIFACT_URL}/benchmark_reports.tar.bz"
+upload_artifact ${WORKSPACE_TMP}/benchmark_reports.tar.bz
 
 exit $(cat ${WORKSPACE_TMP}/exit_status)

ci/scripts/github/pre_benchmark.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,7 +21,7 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 update_conda_env
 
 rapids-logger "Fetching Build artifacts from ${DISPLAY_ARTIFACT_URL}/"
-fetch_s3 "${ARTIFACT_ENDPOINT}/build.tar.bz" "${WORKSPACE_TMP}/build.tar.bz"
+download_artifact "build.tar.bz"
 
 tar xf "${WORKSPACE_TMP}/build.tar.bz"
 

ci/scripts/github/test.sh

@@ -22,8 +22,8 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 update_conda_env
 
 rapids-logger "Fetching Build artifacts from ${DISPLAY_ARTIFACT_URL}/"
-fetch_s3 "${ARTIFACT_ENDPOINT}/dot_cache.tar.bz" "${WORKSPACE_TMP}/dot_cache.tar.bz"
-fetch_s3 "${ARTIFACT_ENDPOINT}/build.tar.bz" "${WORKSPACE_TMP}/build.tar.bz"
+download_artifact "dot_cache.tar.bz"
+download_artifact "build.tar.bz"
 
 tar xf "${WORKSPACE_TMP}/dot_cache.tar.bz"
 tar xf "${WORKSPACE_TMP}/build.tar.bz"
@@ -60,7 +60,7 @@ cd $(dirname ${REPORTS_DIR})
 tar cfj ${WORKSPACE_TMP}/test_reports.tar.bz $(basename ${REPORTS_DIR})
 
 rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/"
-aws s3 cp ${WORKSPACE_TMP}/test_reports.tar.bz "${ARTIFACT_URL}/test_reports.tar.bz"
+upload_artifact ${WORKSPACE_TMP}/test_reports.tar.bz
 
 TEST_RESULTS=$(($CTEST_RESULTS+$PYTEST_RESULTS))
 exit ${TEST_RESULTS}