Use dependencies.yaml to generate environment files

.devcontainer/opt/mrc/bin/post-attach-command.sh

@@ -28,6 +28,6 @@ sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc;
 
 if conda_env_find "${ENV_NAME}" ; \
 
-then mamba env update --name ${ENV_NAME} -f ${MRC_ROOT}/ci/conda/environments/dev_env.yml --prune; \
-else mamba env create --name ${ENV_NAME} -f ${MRC_ROOT}/ci/conda/environments/dev_env.yml; \
+then mamba env update --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-118_arch-x86_64.yaml --prune; \
+else mamba env create --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-118_arch-x86_64.yaml; \
 fi

.github/workflows/pull_request.yml → .github/workflows/pr.yaml

@@ -43,6 +43,18 @@ permissions:
   statuses: none
 
 jobs:
+  pr-builder:
+    needs:
+      - checks
+      - prepare
+      - ci_pipe
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  checks:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      enable_check_generated_files: false
   prepare:
     name: Prepare
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -0,0 +1,11 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+repos:
+      - repo: https://github.com/rapidsai/dependency-file-generator
+        rev: v1.7.1
+        hooks:
+            - id: rapids-dependency-file-generator
+              args: ["--clean"]
+
+default_language_version:
+      python: python3

CONTRIBUTING.md

@@ -85,7 +85,7 @@ cd $MRC_ROOT
 #### Create MRC Conda environment
 ```bash
 # note: `mamba` may be used in place of `conda` for better performance.
-conda env create -n mrc --file $MRC_ROOT/ci/conda/environments/dev_env.yml
+conda env create -n mrc --file $MRC_ROOT/conda/environments/all_cuda-118_arch-x86_64.yaml
 conda activate mrc
 ```
 #### Build MRC

Dockerfile

@@ -35,14 +35,12 @@ RUN --mount=type=cache,target=/var/cache/apt \
     libnuma1 && \
     rm -rf /var/lib/apt/lists/*
 
-COPY ./ci/conda/environments/* /opt/mrc/conda/environments/
+COPY ./conda/environments/all_cuda-118_arch-x86_64.yaml /opt/mrc/conda/environments/all_cuda-118_arch-x86_64.yaml
 
 RUN --mount=type=cache,target=/opt/conda/pkgs,sharing=locked \
     echo "create env: ${PROJ_NAME}" && \
     CONDA_ALWAYS_YES=true \
-    /opt/conda/bin/mamba env create -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/dev_env.yml && \
-    /opt/conda/bin/mamba env update -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/clang_env.yml && \
-    /opt/conda/bin/mamba env update -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/ci_env.yml && \
+    /opt/conda/bin/mamba env create -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/all_cuda-118_arch-x86_64.yaml && \
     chmod -R a+rwX /opt/conda && \
     rm -rf /tmp/conda
 

README.md

@@ -118,7 +118,7 @@ cd $MRC_ROOT
 #### Create MRC Conda Environment
 ```bash
 # note: `mamba` may be used in place of `conda` for better performance.
-conda env create -n mrc-dev --file $MRC_ROOT/ci/conda/environments/dev_env.yml
+conda env create -n mrc-dev --file $MRC_ROOT/conda/environments/all_cuda-118_arch-x86_64.yaml
 conda activate mrc-dev
 ```
 <!-- omit in toc -->

ci/conda/environments/ci_env.yml → ci/check_style.sh

@@ -1,4 +1,5 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,10 +14,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Additional dependencies only needed during a CI build
-name: mrc
-channels:
-    - conda-forge
-dependencies:
-    - codecov=2.1
-    - conda-merge>=0.2
+set -euo pipefail
+
+rapids-logger "Create checks conda environment"
+. /opt/conda/etc/profile.d/conda.sh
+
+rapids-dependency-file-generator \
+  --output conda \
+  --file_key checks \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+rapids-mamba-retry env create --force -f env.yaml -n checks
+conda activate checks
+
+# Run pre-commit checks
+pre-commit run --all-files --show-diff-on-failure

ci/scripts/github/common.sh

@@ -35,9 +35,7 @@ id
 export NUM_PROC=${PARALLEL_LEVEL:-$(nproc)}
 export BUILD_CC=${BUILD_CC:-"gcc"}
 
-export CONDA_ENV_YML="${MRC_ROOT}/ci/conda/environments/dev_env.yml"
-export CONDA_CLANG_ENV_YML="${MRC_ROOT}/ci/conda/environments/clang_env.yml"
-export CONDA_CI_ENV_YML="${MRC_ROOT}/ci/conda/environments/ci_env.yml"
+export CONDA_ENV_YML="${MRC_ROOT}/conda/environments/all_cuda-118_arch-x86_64.yaml"
 
 export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMRC_BUILD_BENCHMARKS=ON -DMRC_BUILD_EXAMPLES=ON -DMRC_BUILD_PYTHON=ON -DMRC_BUILD_TESTS=ON -DMRC_USE_CONDA=ON -DMRC_PYTHON_BUILD_STUBS=ON"
 export CMAKE_BUILD_WITH_CODECOV="-DCMAKE_BUILD_TYPE=Debug -DMRC_ENABLE_CODECOV=ON -DMRC_PYTHON_PERFORM_INSTALL:BOOL=ON -DMRC_PYTHON_INPLACE_BUILD:BOOL=ON"
@@ -83,33 +81,16 @@ function update_conda_env() {
     # Deactivate the environment first before updating
     conda deactivate
 
-    if [[ "${SKIP_CONDA_ENV_UPDATE}" == "" ]]; then
-        # Make sure we have the conda-merge package installed
-        if [[ -z "$(conda list | grep conda-merge)" ]]; then
-            rapids-mamba-retry install -q -n mrc -c conda-forge "conda-merge>=0.2"
-        fi
-    fi
-
-    # Create a temp directory which we store the combined environment file in
-    condatmpdir=$(mktemp -d)
-
-    # Merge the environments together so we can use --prune. Otherwise --prune
-    # will clobber the last env update
-    conda run -n mrc --live-stream conda-merge ${CONDA_ENV_YML} ${CONDA_CLANG_ENV_YML} ${CONDA_CI_ENV_YML} > ${condatmpdir}/merged_env.yml
-
     if [[ "${SKIP_CONDA_ENV_UPDATE}" == "" ]]; then
         # Update the conda env with prune remove excess packages (in case one was removed from the env)
-        rapids-mamba-retry env update -n mrc --prune --file ${condatmpdir}/merged_env.yml
+        rapids-mamba-retry env update -n mrc --prune --file ${CONDA_ENV_YML}
     fi
 
-    # Delete the temp directory
-    rm -rf ${condatmpdir}
-
     # Finally, reactivate
     conda activate mrc
 
     rapids-logger "Final Conda Environment"
-    conda list
+    mamba list
 }
 
 print_env_vars

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -0,0 +1,56 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+channels:
+- conda-forge
+- rapidsai
+- nvidia/label/cuda-11.8.0
+- nvidia
+- rapidsai-nightly
+dependencies:
+- bash-completion
+- benchmark=1.6.0
+- boost-cpp=1.82
+- ccache
+- clang-tools=16
+- clang=16
+- clangdev=16
+- clangxx=16
+- cmake=3.25
+- codecov=2.1
+- cuda-cudart-dev=11.8
+- cuda-nvcc
+- cuda-nvml-dev=11.8
+- cuda-nvrtc-dev=11.8
+- cuda-python=11.8.2
+- cuda-tools=11.8
+- cuda-version=11.8
+- cxx-compiler
+- doxygen=1.9.2
+- flake8
+- gcovr=5.0
+- gdb
+- glog=0.6
+- graphviz=3.0
+- gtest=1.13
+- gxx=11.2
+- include-what-you-use=0.20
+- libclang-cpp=16
+- libclang=16
+- libgrpc=1.54.0
+- libhwloc=2.9.2
+- librmm=23.06
+- llvmdev=16
+- ninja=1.10
+- nlohmann_json=3.9
+- numpy>=1.21
+- pkg-config=0.29
+- pre-commit
+- pybind11-stubgen=0.10
+- pytest
+- pytest-asyncio
+- pytest-timeout
+- python=3.10
+- scikit-build>=0.17
+- ucx=1.14
+- yapf
+name: all_cuda-118_arch-x86_64

conda/environments/ci_cuda-118_arch-x86_64.yaml

@@ -0,0 +1,46 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+channels:
+- conda-forge
+- rapidsai
+- nvidia/label/cuda-11.8.0
+- nvidia
+- rapidsai-nightly
+dependencies:
+- benchmark=1.6.0
+- boost-cpp=1.82
+- ccache
+- cmake=3.25
+- codecov=2.1
+- cuda-cudart-dev=11.8
+- cuda-nvcc
+- cuda-nvml-dev=11.8
+- cuda-nvrtc-dev=11.8
+- cuda-python=11.8.2
+- cuda-tools=11.8
+- cuda-version=11.8
+- cxx-compiler
+- doxygen=1.9.2
+- flake8
+- gcovr=5.0
+- glog=0.6
+- graphviz=3.0
+- gtest=1.13
+- gxx=11.2
+- include-what-you-use=0.20
+- libgrpc=1.54.0
+- libhwloc=2.9.2
+- librmm=23.06
+- ninja=1.10
+- nlohmann_json=3.9
+- pkg-config=0.29
+- pre-commit
+- pybind11-stubgen=0.10
+- pytest
+- pytest-asyncio
+- pytest-timeout
+- python=3.10
+- scikit-build>=0.17
+- ucx=1.14
+- yapf
+name: ci_cuda-118_arch-x86_64