From 839dcbf4c33caa897daf493ae0bfffce71f33892 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Mon, 4 Dec 2023 20:27:50 +0000 Subject: [PATCH 01/17] add cuda-python to dependencies, make conda-forge highest priority --- dependencies.yaml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 966608a19..f78aea21c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -11,11 +11,11 @@ files: - cudatoolkit channels: + - conda-forge - rapidsai - nvidia/label/cuda-11.8.0 - nvidia - rapidsai-nightly - - conda-forge dependencies: @@ -31,21 +31,21 @@ dependencies: packages: - boost-cpp=1.82 - ccache - - cmake=3.24 + - cmake=3.25 - cuda-nvcc - cxx-compiler - glog=0.6 + - gtest=1.13 - gxx=11.2 - libgrpc=1.54.0 - libhwloc=2.9.2 - librmm=23.06 - ninja=1.10 - - ucx=1.14 - nlohmann_json=3.9 - - gtest=1.13 - - scikit-build>=0.17 - pybind11-stubgen=0.10 - python=3.10 + - scikit-build>=0.17 + - ucx=1.14 cudatoolkit: specific: - output_types: [conda] @@ -54,7 +54,8 @@ dependencies: cuda: "11.8" packages: - cuda-cudart-dev=11.8 - - cuda-nvrtc-dev=11.8 - - cuda-version=11.8 - cuda-nvml-dev=11.8 + - cuda-nvrtc-dev=11.8 + - cuda-python=11.8.2 - cuda-tools=11.8 + - cuda-version=11.8 From 8ee6b604fd4a42bd72faa0f0847a02b225a0df8c Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Mon, 4 Dec 2023 22:00:49 +0000 Subject: [PATCH 02/17] convert env yamls to dependencies yaml --- .../opt/mrc/bin/post-attach-command.sh | 4 +- .pre-commit-config.yaml | 11 +++ CONTRIBUTING.md | 2 +- Dockerfile | 6 +- README.md | 2 +- ci/conda/environments/ci_env.yml | 22 ------ ci/conda/environments/clang_env.yml | 29 ------- ci/scripts/github/common.sh | 25 +----- .../all_cuda-118_arch-x86_64.yaml | 51 ++++++++++++ .../environments/ci_cuda-118_arch-x86_64.yaml | 41 ++++++++++ dependencies.yaml | 79 ++++++++++++++++--- 11 files changed, 182 insertions(+), 90 deletions(-) create mode 100644 .pre-commit-config.yaml delete mode 100644 ci/conda/environments/ci_env.yml delete mode 100644 ci/conda/environments/clang_env.yml create mode 100644 conda/environments/all_cuda-118_arch-x86_64.yaml create mode 100644 conda/environments/ci_cuda-118_arch-x86_64.yaml diff --git a/.devcontainer/opt/mrc/bin/post-attach-command.sh b/.devcontainer/opt/mrc/bin/post-attach-command.sh index eb00a5061..ed0ede1eb 100755 --- a/.devcontainer/opt/mrc/bin/post-attach-command.sh +++ b/.devcontainer/opt/mrc/bin/post-attach-command.sh @@ -28,6 +28,6 @@ sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc; if conda_env_find "${ENV_NAME}" ; \ -then mamba env update --name ${ENV_NAME} -f ${MRC_ROOT}/ci/conda/environments/dev_env.yml --prune; \ -else mamba env create --name ${ENV_NAME} -f ${MRC_ROOT}/ci/conda/environments/dev_env.yml; \ +then mamba env update --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-118_arch-x86_64.yaml --prune; \ +else mamba env create --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-118_arch-x86_64.yaml; \ fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..bdfff9d22 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,11 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. + +repos: + - repo: https://github.com/rapidsai/dependency-file-generator + rev: v1.7.1 + hooks: + - id: rapids-dependency-file-generator + args: ["--clean"] + +default_language_version: + python: python3 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b437ba321..4972d8d62 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -85,7 +85,7 @@ cd $MRC_ROOT #### Create MRC Conda environment ```bash # note: `mamba` may be used in place of `conda` for better performance. -conda env create -n mrc --file $MRC_ROOT/ci/conda/environments/dev_env.yml +conda env create -n mrc --file $MRC_ROOT/conda/environments/all_cuda-118_arch-x86_64.yaml conda activate mrc ``` #### Build MRC diff --git a/Dockerfile b/Dockerfile index e303f19f9..b6852f924 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,14 +35,12 @@ RUN --mount=type=cache,target=/var/cache/apt \ libnuma1 && \ rm -rf /var/lib/apt/lists/* -COPY ./ci/conda/environments/* /opt/mrc/conda/environments/ +COPY ./conda/environments/all_cuda-118_arch-x86_64.yaml /opt/mrc/conda/environments/all_cuda-118_arch-x86_64.yaml RUN --mount=type=cache,target=/opt/conda/pkgs,sharing=locked \ echo "create env: ${PROJ_NAME}" && \ CONDA_ALWAYS_YES=true \ - /opt/conda/bin/mamba env create -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/dev_env.yml && \ - /opt/conda/bin/mamba env update -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/clang_env.yml && \ - /opt/conda/bin/mamba env update -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/ci_env.yml && \ + /opt/conda/bin/mamba env create -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/all_cuda-118_arch-x86_64.yaml && \ chmod -R a+rwX /opt/conda && \ rm -rf /tmp/conda diff --git a/README.md b/README.md index 4bdb5c3a3..1da4b6f50 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ cd $MRC_ROOT #### Create MRC Conda Environment ```bash # note: `mamba` may be used in place of `conda` for better performance. -conda env create -n mrc-dev --file $MRC_ROOT/ci/conda/environments/dev_env.yml +conda env create -n mrc-dev --file $MRC_ROOT/conda/environments/all_cuda-118_arch-x86_64.yaml conda activate mrc-dev ``` diff --git a/ci/conda/environments/ci_env.yml b/ci/conda/environments/ci_env.yml deleted file mode 100644 index ad05425dd..000000000 --- a/ci/conda/environments/ci_env.yml +++ /dev/null @@ -1,22 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Additional dependencies only needed during a CI build -name: mrc -channels: - - conda-forge -dependencies: - - codecov=2.1 - - conda-merge>=0.2 diff --git a/ci/conda/environments/clang_env.yml b/ci/conda/environments/clang_env.yml deleted file mode 100644 index bebe11bfd..000000000 --- a/ci/conda/environments/clang_env.yml +++ /dev/null @@ -1,29 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Additional dependencies needed for clang, assumes dependencies from `dev_env.yml` -# or `dev_env_nogcc.yml` has already been installed -name: mrc -channels: - - conda-forge -dependencies: - - clang=16 - - clang-tools=16 - - clangdev=16 - - clangxx=16 - - libclang=16 - - libclang-cpp=16 - - llvmdev=16 - - include-what-you-use=0.20 diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh index 17807bdce..e67dc7a35 100644 --- a/ci/scripts/github/common.sh +++ b/ci/scripts/github/common.sh @@ -35,9 +35,7 @@ id export NUM_PROC=${PARALLEL_LEVEL:-$(nproc)} export BUILD_CC=${BUILD_CC:-"gcc"} -export CONDA_ENV_YML="${MRC_ROOT}/ci/conda/environments/dev_env.yml" -export CONDA_CLANG_ENV_YML="${MRC_ROOT}/ci/conda/environments/clang_env.yml" -export CONDA_CI_ENV_YML="${MRC_ROOT}/ci/conda/environments/ci_env.yml" +export CONDA_ENV_YML="${MRC_ROOT}/conda/environments/all_cuda-118_arch-x86_64.yaml" export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMRC_BUILD_BENCHMARKS=ON -DMRC_BUILD_EXAMPLES=ON -DMRC_BUILD_PYTHON=ON -DMRC_BUILD_TESTS=ON -DMRC_USE_CONDA=ON -DMRC_PYTHON_BUILD_STUBS=ON" export CMAKE_BUILD_WITH_CODECOV="-DCMAKE_BUILD_TYPE=Debug -DMRC_ENABLE_CODECOV=ON -DMRC_PYTHON_PERFORM_INSTALL:BOOL=ON -DMRC_PYTHON_INPLACE_BUILD:BOOL=ON" @@ -83,33 +81,16 @@ function update_conda_env() { # Deactivate the environment first before updating conda deactivate - if [[ "${SKIP_CONDA_ENV_UPDATE}" == "" ]]; then - # Make sure we have the conda-merge package installed - if [[ -z "$(conda list | grep conda-merge)" ]]; then - rapids-mamba-retry install -q -n mrc -c conda-forge "conda-merge>=0.2" - fi - fi - - # Create a temp directory which we store the combined environment file in - condatmpdir=$(mktemp -d) - - # Merge the environments together so we can use --prune. Otherwise --prune - # will clobber the last env update - conda run -n mrc --live-stream conda-merge ${CONDA_ENV_YML} ${CONDA_CLANG_ENV_YML} ${CONDA_CI_ENV_YML} > ${condatmpdir}/merged_env.yml - if [[ "${SKIP_CONDA_ENV_UPDATE}" == "" ]]; then # Update the conda env with prune remove excess packages (in case one was removed from the env) - rapids-mamba-retry env update -n mrc --prune --file ${condatmpdir}/merged_env.yml + rapids-mamba-retry env update -n mrc --prune --file ${CONDA_ENV_YML} fi - # Delete the temp directory - rm -rf ${condatmpdir} - # Finally, reactivate conda activate mrc rapids-logger "Final Conda Environment" - conda list + mamba list } print_env_vars diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml new file mode 100644 index 000000000..430dc8d52 --- /dev/null +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -0,0 +1,51 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- conda-forge +- rapidsai +- nvidia/label/cuda-11.8.0 +- nvidia +- rapidsai-nightly +dependencies: +- bash-completion +- benchmark=1.6.0 +- boost-cpp=1.82 +- ccache +- clang-tools=16 +- clang=16 +- clangdev=16 +- clangxx=16 +- cmake=3.25 +- codecov=2.1 +- cuda-cudart-dev=11.8 +- cuda-nvcc +- cuda-nvml-dev=11.8 +- cuda-nvrtc-dev=11.8 +- cuda-python=11.8.2 +- cuda-tools=11.8 +- cuda-version=11.8 +- cxx-compiler +- flake8 +- gdb +- glog=0.6 +- gtest=1.13 +- gxx=11.2 +- include-what-you-use=0.20 +- libclang-cpp=16 +- libclang=16 +- libgrpc=1.54.0 +- libhwloc=2.9.2 +- librmm=23.06 +- llvmdev=16 +- ninja=1.10 +- nlohmann_json=3.9 +- numpy>=1.21 +- pybind11-stubgen=0.10 +- pytest +- pytest-asyncio +- pytest-timeout +- python=3.10 +- scikit-build>=0.17 +- ucx=1.14 +- yapf +name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/ci_cuda-118_arch-x86_64.yaml b/conda/environments/ci_cuda-118_arch-x86_64.yaml new file mode 100644 index 000000000..49096bbbf --- /dev/null +++ b/conda/environments/ci_cuda-118_arch-x86_64.yaml @@ -0,0 +1,41 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- conda-forge +- rapidsai +- nvidia/label/cuda-11.8.0 +- nvidia +- rapidsai-nightly +dependencies: +- benchmark=1.6.0 +- boost-cpp=1.82 +- ccache +- cmake=3.25 +- codecov=2.1 +- cuda-cudart-dev=11.8 +- cuda-nvcc +- cuda-nvml-dev=11.8 +- cuda-nvrtc-dev=11.8 +- cuda-python=11.8.2 +- cuda-tools=11.8 +- cuda-version=11.8 +- cxx-compiler +- flake8 +- glog=0.6 +- gtest=1.13 +- gxx=11.2 +- include-what-you-use=0.20 +- libgrpc=1.54.0 +- libhwloc=2.9.2 +- librmm=23.06 +- ninja=1.10 +- nlohmann_json=3.9 +- pybind11-stubgen=0.10 +- pytest +- pytest-asyncio +- pytest-timeout +- python=3.10 +- scikit-build>=0.17 +- ucx=1.14 +- yapf +name: ci_cuda-118_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index f78aea21c..376ba027e 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -6,8 +6,26 @@ files: cuda: ["11.8"] arch: [x86_64] includes: - - empty - - build_cpp + - build + - developer_productivity + - code_style + - testing + - benchmarking + - ci + - examples + - cudatoolkit + + ci: + output: conda + matrix: + cuda: ["11.8"] + arch: [x86_64] + includes: + - build + - code_style + - testing + - benchmarking + - ci - cudatoolkit channels: @@ -19,13 +37,7 @@ channels: dependencies: - empty: - common: - - output_types: [conda] - packages: - - cxx-compiler - - build_cpp: + build: common: - output_types: [conda] packages: @@ -46,6 +58,55 @@ dependencies: - python=3.10 - scikit-build>=0.17 - ucx=1.14 + + developer_productivity: + common: + - output_types: [conda] + packages: + - clang=16 + - clang-tools=16 + - clangdev=16 + - clangxx=16 + - libclang=16 + - libclang-cpp=16 + - llvmdev=16 + - gdb + - bash-completion + + code_style: + common: + - output_types: [conda] + packages: + - flake8 + - yapf + - include-what-you-use=0.20 + + testing: + common: + - output_types: [conda] + packages: + - pytest + - pytest-timeout + - pytest-asyncio + + benchmarking: + common: + - output_types: [conda] + packages: + - benchmark=1.6.0 + + ci: + common: + - output_types: [conda] + packages: + - codecov=2.1 + + examples: + common: + - output_types: [conda] + packages: + - numpy>=1.21 + cudatoolkit: specific: - output_types: [conda] From bd96210f688a6ebcb6cdaa532ea238048349905e Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Mon, 4 Dec 2023 22:05:23 +0000 Subject: [PATCH 03/17] add pre-commit to dependencies --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/environments/ci_cuda-118_arch-x86_64.yaml | 1 + dependencies.yaml | 10 ++++++---- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 430dc8d52..d84cc996f 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -40,6 +40,7 @@ dependencies: - ninja=1.10 - nlohmann_json=3.9 - numpy>=1.21 +- pre-commit - pybind11-stubgen=0.10 - pytest - pytest-asyncio diff --git a/conda/environments/ci_cuda-118_arch-x86_64.yaml b/conda/environments/ci_cuda-118_arch-x86_64.yaml index 49096bbbf..340fbadd2 100644 --- a/conda/environments/ci_cuda-118_arch-x86_64.yaml +++ b/conda/environments/ci_cuda-118_arch-x86_64.yaml @@ -30,6 +30,7 @@ dependencies: - librmm=23.06 - ninja=1.10 - nlohmann_json=3.9 +- pre-commit - pybind11-stubgen=0.10 - pytest - pytest-asyncio diff --git a/dependencies.yaml b/dependencies.yaml index 376ba027e..6b6559ab0 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -63,15 +63,16 @@ dependencies: common: - output_types: [conda] packages: - - clang=16 + - bash-completion - clang-tools=16 + - clang=16 - clangdev=16 - clangxx=16 - - libclang=16 + - gdb - libclang-cpp=16 + - libclang=16 - llvmdev=16 - - gdb - - bash-completion + - pre-commit code_style: common: @@ -100,6 +101,7 @@ dependencies: - output_types: [conda] packages: - codecov=2.1 + - pre-commit examples: common: From 3ac34b3e80e8f91d8a488c05502c8c1d80d29688 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Mon, 4 Dec 2023 22:57:05 +0000 Subject: [PATCH 04/17] add pkg-config to dependencies --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/environments/ci_cuda-118_arch-x86_64.yaml | 1 + dependencies.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index d84cc996f..00ebba42a 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -40,6 +40,7 @@ dependencies: - ninja=1.10 - nlohmann_json=3.9 - numpy>=1.21 +- pkg-config=0.29 - pre-commit - pybind11-stubgen=0.10 - pytest diff --git a/conda/environments/ci_cuda-118_arch-x86_64.yaml b/conda/environments/ci_cuda-118_arch-x86_64.yaml index 340fbadd2..a6f1286a1 100644 --- a/conda/environments/ci_cuda-118_arch-x86_64.yaml +++ b/conda/environments/ci_cuda-118_arch-x86_64.yaml @@ -30,6 +30,7 @@ dependencies: - librmm=23.06 - ninja=1.10 - nlohmann_json=3.9 +- pkg-config=0.29 - pre-commit - pybind11-stubgen=0.10 - pytest diff --git a/dependencies.yaml b/dependencies.yaml index 6b6559ab0..f42087106 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -54,6 +54,7 @@ dependencies: - librmm=23.06 - ninja=1.10 - nlohmann_json=3.9 + - pkg-config=0.29 - pybind11-stubgen=0.10 - python=3.10 - scikit-build>=0.17 From 9849ecc656b2bd49bc4b7bb348bc2fd339d42380 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Mon, 4 Dec 2023 23:20:41 +0000 Subject: [PATCH 05/17] add gcovr and doxygen to dependencies --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 ++ conda/environments/ci_cuda-118_arch-x86_64.yaml | 2 ++ dependencies.yaml | 9 +++++++++ 3 files changed, 13 insertions(+) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 00ebba42a..6fefcaf31 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -25,7 +25,9 @@ dependencies: - cuda-tools=11.8 - cuda-version=11.8 - cxx-compiler +- doxygen=1.9.2 - flake8 +- gcovr=5.0 - gdb - glog=0.6 - gtest=1.13 diff --git a/conda/environments/ci_cuda-118_arch-x86_64.yaml b/conda/environments/ci_cuda-118_arch-x86_64.yaml index a6f1286a1..823b25ce7 100644 --- a/conda/environments/ci_cuda-118_arch-x86_64.yaml +++ b/conda/environments/ci_cuda-118_arch-x86_64.yaml @@ -20,7 +20,9 @@ dependencies: - cuda-tools=11.8 - cuda-version=11.8 - cxx-compiler +- doxygen=1.9.2 - flake8 +- gcovr=5.0 - glog=0.6 - gtest=1.13 - gxx=11.2 diff --git a/dependencies.yaml b/dependencies.yaml index f42087106..dc5d9971f 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -13,6 +13,7 @@ files: - benchmarking - ci - examples + - documentation - cudatoolkit ci: @@ -26,6 +27,7 @@ files: - testing - benchmarking - ci + - documentation - cudatoolkit channels: @@ -102,6 +104,7 @@ dependencies: - output_types: [conda] packages: - codecov=2.1 + - gcovr=5.0 - pre-commit examples: @@ -110,6 +113,12 @@ dependencies: packages: - numpy>=1.21 + documentation: + common: + - output_types: [conda] + packages: + - doxygen=1.9.2 + cudatoolkit: specific: - output_types: [conda] From a7bb74352034c64646b5f0d3b36cce31926e6a24 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Tue, 5 Dec 2023 00:06:09 +0000 Subject: [PATCH 06/17] add graphviz to dependencies --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/environments/ci_cuda-118_arch-x86_64.yaml | 1 + dependencies.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 6fefcaf31..313a30ff4 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -30,6 +30,7 @@ dependencies: - gcovr=5.0 - gdb - glog=0.6 +- graphviz=3.0 - gtest=1.13 - gxx=11.2 - include-what-you-use=0.20 diff --git a/conda/environments/ci_cuda-118_arch-x86_64.yaml b/conda/environments/ci_cuda-118_arch-x86_64.yaml index 823b25ce7..30288a878 100644 --- a/conda/environments/ci_cuda-118_arch-x86_64.yaml +++ b/conda/environments/ci_cuda-118_arch-x86_64.yaml @@ -24,6 +24,7 @@ dependencies: - flake8 - gcovr=5.0 - glog=0.6 +- graphviz=3.0 - gtest=1.13 - gxx=11.2 - include-what-you-use=0.20 diff --git a/dependencies.yaml b/dependencies.yaml index dc5d9971f..4d81d058c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -118,6 +118,7 @@ dependencies: - output_types: [conda] packages: - doxygen=1.9.2 + - graphviz=3.0 cudatoolkit: specific: From f967fc64f50dcb72b5c3f13d4795cdc39c6615c8 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 6 Dec 2023 17:43:13 +0000 Subject: [PATCH 07/17] add pre-commit to ci checks --- .github/workflows/pull_request.yml | 5 +++++ ci/check_style.sh | 18 ++++++++++++++++++ dependencies.yaml | 14 +++++++++++++- 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 ci/check_style.sh diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index dd5b73dd3..acb67bd32 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -80,3 +80,8 @@ jobs: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} CONDA_TOKEN: ${{ secrets.CONDA_TOKEN }} NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + checks: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.02 + with: + enable_check_generated_files: false diff --git a/ci/check_style.sh b/ci/check_style.sh new file mode 100644 index 000000000..0ee6e88e5 --- /dev/null +++ b/ci/check_style.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright (c) 2020-2023, NVIDIA CORPORATION. + +set -euo pipefail + +rapids-logger "Create checks conda environment" +. /opt/conda/etc/profile.d/conda.sh + +rapids-dependency-file-generator \ + --output conda \ + --file_key checks \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +rapids-mamba-retry env create --force -f env.yaml -n checks +conda activate checks + +# Run pre-commit checks +pre-commit run --all-files --show-diff-on-failure diff --git a/dependencies.yaml b/dependencies.yaml index 4d81d058c..8d6f0d3b2 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -7,6 +7,7 @@ files: arch: [x86_64] includes: - build + - checks - developer_productivity - code_style - testing @@ -30,6 +31,12 @@ files: - documentation - cudatoolkit + checks: + output: none + includes: + - checks + + channels: - conda-forge - rapidsai @@ -62,6 +69,12 @@ dependencies: - scikit-build>=0.17 - ucx=1.14 + checks: + common: + - output_types: [conda] + packages: + - pre-commit + developer_productivity: common: - output_types: [conda] @@ -75,7 +88,6 @@ dependencies: - libclang-cpp=16 - libclang=16 - llvmdev=16 - - pre-commit code_style: common: From b154f374f3518f87a58de6e5ac059e8236eae313 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 6 Dec 2023 17:46:28 +0000 Subject: [PATCH 08/17] make check_style.sh executable --- ci/check_style.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/check_style.sh diff --git a/ci/check_style.sh b/ci/check_style.sh old mode 100644 new mode 100755 From e47c26da5fd35adc585a3ba0714647ed34abe044 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 6 Dec 2023 17:53:02 +0000 Subject: [PATCH 09/17] rename pull_request.yml --- .github/workflows/{pull_request.yml => pr.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{pull_request.yml => pr.yml} (100%) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pr.yml similarity index 100% rename from .github/workflows/pull_request.yml rename to .github/workflows/pr.yml From 9f1227d8ec48db155119a96aea37fd2acdb2a1bb Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 6 Dec 2023 17:55:18 +0000 Subject: [PATCH 10/17] . --- .github/workflows/{pr.yml => pr.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{pr.yml => pr.yaml} (100%) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yaml similarity index 100% rename from .github/workflows/pr.yml rename to .github/workflows/pr.yaml From a531cafd654d7880e58c2affe425caf65f2957ee Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 6 Dec 2023 17:58:25 +0000 Subject: [PATCH 11/17] add pr-builder job to pr.yaml --- .github/workflows/pr.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index acb67bd32..3f1697ba8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -43,6 +43,9 @@ permissions: statuses: none jobs: + pr-builder: + needs: + - checks prepare: name: Prepare runs-on: ubuntu-latest From abe0da66e93c9c9edf9cdfb629798af4563e3252 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 6 Dec 2023 18:02:24 +0000 Subject: [PATCH 12/17] . --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3f1697ba8..86d8121f9 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -83,6 +83,7 @@ jobs: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} CONDA_TOKEN: ${{ secrets.CONDA_TOKEN }} NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + checks: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.02 From a34bf0d6cccd5f3ff5c84292441449cbd20b8a2e Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 6 Dec 2023 18:20:58 +0000 Subject: [PATCH 13/17] fix pr-builder job --- .github/workflows/pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 86d8121f9..77b7ffaad 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -46,6 +46,8 @@ jobs: pr-builder: needs: - checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.02 prepare: name: Prepare runs-on: ubuntu-latest From 643f9f0f4fbd14dc1c753be90bbe33617451e777 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 6 Dec 2023 18:22:23 +0000 Subject: [PATCH 14/17] fix pr-builder job --- .github/workflows/pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 77b7ffaad..53da3a647 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -46,6 +46,8 @@ jobs: pr-builder: needs: - checks + - prepare + - pi_pipe secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.02 prepare: From 40c7d17624ba8d73369720edef07f12d6e3c7c66 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 6 Dec 2023 18:22:49 +0000 Subject: [PATCH 15/17] fix pr-builder job --- .github/workflows/pr.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 53da3a647..904617411 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -50,6 +50,11 @@ jobs: - pi_pipe secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.02 + checks: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.02 + with: + enable_check_generated_files: false prepare: name: Prepare runs-on: ubuntu-latest @@ -87,9 +92,3 @@ jobs: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} CONDA_TOKEN: ${{ secrets.CONDA_TOKEN }} NGC_API_KEY: ${{ secrets.NGC_API_KEY }} - - checks: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.02 - with: - enable_check_generated_files: false From 77684ea982beb031867494c2e26e1af250e83487 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 6 Dec 2023 18:23:37 +0000 Subject: [PATCH 16/17] fix pr-builder job --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 904617411..1ada0ea1b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -47,7 +47,7 @@ jobs: needs: - checks - prepare - - pi_pipe + - ci_pipe secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.02 checks: From 7231a5e757a479b268aef3b499c5ed551a6b8922 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 6 Dec 2023 18:34:05 +0000 Subject: [PATCH 17/17] add copyright header to `check_style.sh` --- ci/check_style.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ci/check_style.sh b/ci/check_style.sh index 0ee6e88e5..6ef101616 100755 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -1,5 +1,18 @@ #!/bin/bash -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. set -euo pipefail