Schedule Daily #155
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Schedule Daily | |
| on: | |
| schedule: | |
| - cron: "0 1 * * *" | |
| workflow_dispatch: | |
| env: | |
| AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }} | |
| BAZEL_STARTUP_ARGS: "--output_base=/var/tmp/bazel-output/" | |
| CI_COMMIT_SHA: ${{ github.sha }} | |
| CI_COMMIT_REF_PROTECTED: ${{ github.ref_protected }} | |
| CI_JOB_NAME: ${{ github.job }} | |
| CI_JOB_ID: ${{ github.job }} # github does not expose this variable https://github.com/orgs/community/discussions/8945 | |
| CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| CI_PIPELINE_SOURCE: ${{ github.event_name }} | |
| CI_PROJECT_DIR: ${{ github.workspace }} | |
| CI_MERGE_REQUEST_TARGET_BRANCH_NAME: ${{ github.event.pull_request.base.ref }} | |
| ROOT_PIPELINE_ID: ${{ github.run_id }} | |
| DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }} | |
| DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }} | |
| BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }} | |
| BUILDEVENT_DATASET: "github-ci-dfinity" | |
| jobs: | |
| cut-release-candidate: | |
| name: Cut RC | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/master') | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| token: ${{ secrets.IDX_PUSH_TO_PR }} | |
| - name: Cut Release Candidate | |
| shell: bash | |
| run: | | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "IDX GitLab Automation" | |
| RC_BRANCH_NAME="rc--$(date '+%Y-%m-%d_%H-%M')" | |
| git switch --force-create "$RC_BRANCH_NAME" HEAD | |
| git push --force --set-upstream origin "$RC_BRANCH_NAME" | |
| rust-benchmarks: | |
| name: Bazel Run Rust Benchmarks | |
| runs-on: | |
| # see linux-x86-64 runner group | |
| labels: rust-benchmarks | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:26cc347efa50935342742acddfb5d710fae1982d401911013ad8750f0603c590 | |
| # running on bare metal machine using ubuntu user | |
| options: --user ubuntu -v /cache:/cache | |
| timeout-minutes: 720 # 12 hours | |
| strategy: | |
| matrix: | |
| targets: | |
| - "//rs/crypto/..." | |
| - "//rs/state_manager/..." | |
| - "//rs/certification/..." | |
| - "//rs/boundary_node/ic_boundary/..." | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: ./gitlab-ci/src/ci-scripts/before-script.sh | |
| - name: Run Rust Benchmarks | |
| id: rust-benchmarks | |
| shell: bash | |
| run: | | |
| ./gitlab-ci/src/ci-scripts/rust-benchmarks.sh | |
| env: | |
| BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel" | |
| BAZEL_COMMAND: "run" | |
| BAZEL_STARTUP_ARGS: "--output_base=/var/tmp/bazel-output/" | |
| RUSTFLAGS: "--remap-path-prefix=${CI_PROJECT_DIR}=/ic" | |
| RUST_BACKTRACE: "full" | |
| TARGETS: ${{ matrix.targets }} | |
| bazel-test-bare-metal: | |
| name: Bazel Test Bare Metal | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:26cc347efa50935342742acddfb5d710fae1982d401911013ad8750f0603c590 | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp | |
| timeout-minutes: 120 | |
| runs-on: | |
| group: zh1 | |
| labels: dind-large | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: ./gitlab-ci/src/ci-scripts/before-script.sh | |
| - name: Run Bazel Launch Bare Metal | |
| shell: bash | |
| run: | | |
| echo "$ZH2_DLL01_CSV_SECRETS" > file1 | |
| echo "$ZH2_FILE_SHARE_KEY" > file2 && chmod 400 file2 | |
| # shellcheck disable=SC2046,SC2086 | |
| bazel ${BAZEL_STARTUP_ARGS} run ${BAZEL_CI_CONFIG} \ | |
| //ic-os/setupos/envs/dev:launch_bare_metal -- \ | |
| --config_path "$(realpath ./ic-os/dev-tools/bare_metal_deployment/zh2-dll01.yaml)" \ | |
| --csv_filename "$(realpath file1)" \ | |
| --file_share_ssh_key "$(realpath file2)" \ | |
| --file_share_username ci_interim \ | |
| --ci_mode | |
| bazel clean | |
| env: | |
| BAZEL_STARTUP_ARGS: "--output_base=/var/tmp/bazel-output/" | |
| BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel" | |
| ZH2_DLL01_CSV_SECRETS: "${{ secrets.ZH2_DLL01_CSV_SECRETS }}" | |
| ZH2_FILE_SHARE_KEY: "${{ secrets.ZH2_FILE_SHARE_KEY }}" | |
| nns-tests-nightly: | |
| name: Bazel Test NNS Nightly | |
| runs-on: | |
| group: zh1 | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:26cc347efa50935342742acddfb5d710fae1982d401911013ad8750f0603c590 | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: ./gitlab-ci/src/ci-scripts/before-script.sh | |
| - name: Run NNS Tests Nightly | |
| id: bazel-test-all | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "test" | |
| BAZEL_TARGETS: "//rs/nns/..." | |
| BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel" | |
| BAZEL_EXTRA_ARGS: "--keep_going --test_tag_filters=nns_tests_nightly --test_env=SSH_AUTH_SOCK --test_env=NNS_CANISTER_UPGRADE_SEQUENCE=all" | |
| HONEYCOMB_API_TOKEN: ${{ secrets.HONEYCOMB_API_TOKEN }} | |
| SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} | |
| system-tests-benchmarks-nightly: | |
| name: Bazel System Test Benchmarks | |
| runs-on: | |
| group: zh1 | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:26cc347efa50935342742acddfb5d710fae1982d401911013ad8750f0603c590 | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp | |
| timeout-minutes: 480 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: ./gitlab-ci/src/ci-scripts/before-script.sh | |
| - name: Set Benchmark Targets | |
| shell: bash | |
| run: | | |
| set -xeuo pipefail | |
| echo "BENCHMARK_TARGETS=$(bazel query 'attr(tags, system_test_benchmark, //rs/...)')" >> $GITHUB_ENV | |
| - name: Test System Test Benchmarks | |
| id: bazel-system-test-benchmarks | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "test" | |
| BAZEL_TARGETS: ${{ env.BENCHMARK_TARGETS }} | |
| BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel" | |
| # note: there's just one performance cluster, so the job can't be parallelized | |
| BAZEL_EXTRA_ARGS: "--//bazel:enable_upload_perf_systest_results=True --keep_going --jobs 1" | |
| HONEYCOMB_API_TOKEN: ${{ secrets.HONEYCOMB_API_TOKEN }} | |
| dependency-scan-nightly: | |
| name: Dependency Scan Nightly | |
| runs-on: | |
| group: zh1 | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:26cc347efa50935342742acddfb5d710fae1982d401911013ad8750f0603c590 | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp | |
| timeout-minutes: 60 | |
| permissions: | |
| actions: write | |
| env: | |
| SHELL_WRAPPER: "/usr/bin/time" | |
| CARGO_WASMPACK_VERSION: "0.12.1" | |
| DEFAULT_NODE_VERSION: "20" | |
| CI_PROJECT_PATH: ${{ github.repository }} | |
| CI_PIPELINE_ID: ${{ github.run_id }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} | |
| SLACK_PSEC_BOT_OAUTH_TOKEN: ${{ secrets.SLACK_PSEC_BOT_OAUTH_TOKEN }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: ./gitlab-ci/src/ci-scripts/before-script.sh | |
| - name: Setup environment deps | |
| id: setup-environment-deps | |
| shell: bash | |
| run: | | |
| pip3 install --ignore-installed -r requirements.txt | |
| cargo install wasm-pack --version "${CARGO_WASMPACK_VERSION}" | |
| source "${NVM_DIR}/nvm.sh" | |
| nvm use ${DEFAULT_NODE_VERSION} | |
| node --version | |
| npm --version | |
| - name: Run Dependency Scan Nightly | |
| id: dependency-scan-nightly | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| export PYTHONPATH=$PWD/gitlab-ci/src:$PWD/gitlab-ci/src/dependencies | |
| cd gitlab-ci/src/dependencies/ | |
| $SHELL_WRAPPER python3 job/bazel_rust_ic_scanner_periodic_job.py | |
| $SHELL_WRAPPER python3 job/npm_scanner_periodic_job.py | |
| $SHELL_WRAPPER python3 job/bazel_trivy_container_ic_scanner_periodic_job.py |