From 4d16f01c9d969e726a9efc6debd19dcd968bdf01 Mon Sep 17 00:00:00 2001 From: Justin Variath Thomas Date: Thu, 26 Oct 2023 15:33:33 -0400 Subject: [PATCH] WX-1307 Azure E2E test (#7239) Co-authored-by: Janet Gainer-Dewar --- .github/workflows/azure_e2e_run_workflow.yml | 136 +++++++++++++++++++ .gitignore | 6 + 2 files changed, 142 insertions(+) create mode 100644 .github/workflows/azure_e2e_run_workflow.yml diff --git a/.github/workflows/azure_e2e_run_workflow.yml b/.github/workflows/azure_e2e_run_workflow.yml new file mode 100644 index 00000000000..d2df9b38441 --- /dev/null +++ b/.github/workflows/azure_e2e_run_workflow.yml @@ -0,0 +1,136 @@ +name: 'Azure e2e - Run Workflow' +on: + schedule: + - cron: '0 16 * * *' # UTC 4pm, EST 11am, EDT 12pm + workflow_dispatch: + inputs: + target-branch: + description: 'Branch name of dsp-reusable-workflows repo to run tests on' + required: true + default: 'main' + type: string + owner-subject: + description: 'Owner of billing project' + required: true + default: 'hermione.owner@quality.firecloud.org' + type: string + service-account: + description: 'Email address or unique identifier of the Google Cloud service account for which to generate credentials' + required: true + default: 'firecloud-qa@broad-dsde-qa.iam.gserviceaccount.com' + type: string + +env: + BROADBOT_TOKEN: '${{ secrets.BROADBOT_GITHUB_TOKEN }}' # github token for access to kick off a job in the private repo + RUN_NAME_SUFFIX: '${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}' + +jobs: + + # This job provisions useful parameters for e2e tests + params-gen: + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + outputs: + project-name: ${{ steps.gen.outputs.project_name }} + bee-name: '${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt}}-dev' + steps: + - name: Generate a random billing project name + id: 'gen' + run: | + project_name=$(echo "tmp-billing-project-$(uuidgen)" | cut -c -30) + echo "project_name=${project_name}" >> $GITHUB_OUTPUT + + create-bee-workflow: + runs-on: ubuntu-latest + needs: [params-gen] + permissions: + contents: 'read' + id-token: 'write' + steps: + - name: Dispatch to terra-github-workflows + uses: broadinstitute/workflow-dispatch@v3 + with: + workflow: bee-create + repo: broadinstitute/terra-github-workflows + ref: refs/heads/main + token: ${{ env.BROADBOT_TOKEN }} + # NOTE: Opting to use "prod" instead of custom tag since I specifically want to test against the current prod state + # NOTE: For testing/development purposes I'm using dev + inputs: '{ "bee-name": "${{ needs.params-gen.outputs.bee-name }}", "version-template": "dev", "bee-template-name": "rawls-e2e-azure-tests"}' + + create-and-attach-billing-project-to-landing-zone-workflow: + runs-on: ubuntu-latest + needs: [create-bee-workflow, params-gen] + steps: + - name: dispatch to terra-github-workflows + uses: broadinstitute/workflow-dispatch@v3 + with: + workflow: attach-billing-project-to-landing-zone.yaml + repo: broadinstitute/terra-github-workflows + ref: refs/heads/main + token: ${{ env.BROADBOT_TOKEN }} + inputs: '{ + "run-name": "attach-billing-project-to-landing-zone-${{ env.RUN_NAME_SUFFIX }}", + "bee-name": "${{ needs.params-gen.outputs.bee-name }}", + "billing-project": "${{ needs.params-gen.outputs.project-name }}", + "billing-project-creator": "${{ inputs.owner-subject }}", + "service-account": "${{inputs.service-account}}" }' + + run-cromwell-az-e2e: + needs: [params-gen, create-and-attach-billing-project-to-landing-zone-workflow] + permissions: + contents: read + id-token: write + uses: "broadinstitute/dsp-reusable-workflows/.github/workflows/cromwell-az-e2e-test.yaml@main" + with: + branch: "${{ inputs.target-branch }}" + bee-name: "${{ needs.params-gen.outputs.bee-name }}" + billing-project-name: "${{ needs.params-gen.outputs.project-name }}" + + delete-billing-project-v2-from-bee-workflow: + continue-on-error: true + runs-on: ubuntu-latest + needs: [run-cromwell-az-e2e, create-and-attach-billing-project-to-landing-zone-workflow, params-gen] + if: always() + steps: + - name: dispatch to terra-github-workflows + uses: broadinstitute/workflow-dispatch@v3 + with: + workflow: .github/workflows/delete-billing-project-v2-from-bee.yaml + repo: broadinstitute/terra-github-workflows + ref: refs/heads/main + token: ${{ env.BROADBOT_TOKEN }} + inputs: '{ + "run-name": "delete-billing-project-v2-from-bee-${{ env.RUN_NAME_SUFFIX }}", + "bee-name": "${{ needs.params-gen.outputs.bee-name }}", + "billing-project": "${{ needs.params-gen.outputs.project-name }}", + "billing-project-owner": "${{ inputs.owner-subject }}", + "service-account": "${{ inputs.service-account }}", + "silent-on-failure": "false" }' + + destroy-bee-workflow: + runs-on: ubuntu-latest + needs: [params-gen, create-bee-workflow, delete-billing-project-v2-from-bee-workflow] + if: always() + permissions: + contents: 'read' + id-token: 'write' + steps: + - name: dispatch to terra-github-workflows + uses: broadinstitute/workflow-dispatch@v3 + with: + workflow: bee-destroy.yaml + repo: broadinstitute/terra-github-workflows + ref: refs/heads/main + token: ${{ env.BROADBOT_TOKEN }} + inputs: '{ "bee-name": "${{ needs.params-gen.outputs.bee-name }}" }' + wait-for-completion: true + + report-workflow: + uses: broadinstitute/sherlock/.github/workflows/client-report-workflow.yaml@main + with: + notify-slack-channels-upon-workflow-failure: "#cromwell_jenkins_ci_errors" + permissions: + id-token: write diff --git a/.gitignore b/.gitignore index a5b72f6b263..250b6aa3c16 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,9 @@ console_output.txt expected.json run_mode_metadata.json +#bloop files +/.bloop + # custom config cromwell-executions cromwell-test-executions @@ -55,3 +58,6 @@ tesk_application.conf **/venv/ exome_germline_single_sample_v1.3/ **/*.pyc + +# GHA credentials +gha-creds-*.json