Merge branch 'develop' into WX-1260

.github/workflows/azure_e2e_run_workflow.yml

@@ -0,0 +1,136 @@
+name: 'Azure e2e - Run Workflow'
+on:
+  schedule:
+    - cron: '0 16 * * *' # UTC 4pm, EST 11am, EDT 12pm
+  workflow_dispatch:
+    inputs:
+      target-branch:
+        description: 'Branch name of dsp-reusable-workflows repo to run tests on'
+        required: true
+        default: 'main'
+        type: string
+      owner-subject:
+        description: 'Owner of billing project'
+        required: true
+        default: '[email protected]'
+        type: string
+      service-account:
+        description: 'Email address or unique identifier of the Google Cloud service account for which to generate credentials'
+        required: true
+        default: '[email protected]'
+        type: string
+
+env:
+  BROADBOT_TOKEN: '${{ secrets.BROADBOT_GITHUB_TOKEN }}' # github token for access to kick off a job in the private repo
+  RUN_NAME_SUFFIX: '${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
+
+jobs:
+
+  # This job provisions useful parameters for e2e tests
+  params-gen:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    outputs:
+      project-name: ${{ steps.gen.outputs.project_name }}
+      bee-name: '${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt}}-dev'
+    steps:
+      - name: Generate a random billing project name
+        id: 'gen'
+        run: |
+          project_name=$(echo "tmp-billing-project-$(uuidgen)" | cut -c -30)
+          echo "project_name=${project_name}" >> $GITHUB_OUTPUT
+  
+  create-bee-workflow:
+    runs-on: ubuntu-latest
+    needs: [params-gen]
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - name: Dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: bee-create
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.BROADBOT_TOKEN }}
+          # NOTE: Opting to use "prod" instead of custom tag since I specifically want to test against the current prod state
+          # NOTE: For testing/development purposes I'm using dev
+          inputs: '{ "bee-name": "${{ needs.params-gen.outputs.bee-name }}", "version-template": "dev", "bee-template-name": "rawls-e2e-azure-tests"}'
+
+  create-and-attach-billing-project-to-landing-zone-workflow:
+    runs-on: ubuntu-latest
+    needs: [create-bee-workflow, params-gen]
+    steps:
+      - name: dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: attach-billing-project-to-landing-zone.yaml
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.BROADBOT_TOKEN }}
+          inputs: '{
+            "run-name": "attach-billing-project-to-landing-zone-${{ env.RUN_NAME_SUFFIX }}",
+            "bee-name": "${{ needs.params-gen.outputs.bee-name }}",
+            "billing-project": "${{ needs.params-gen.outputs.project-name }}",
+            "billing-project-creator": "${{ inputs.owner-subject }}",
+            "service-account": "${{inputs.service-account}}" }'
+
+  run-cromwell-az-e2e:
+    needs: [params-gen, create-and-attach-billing-project-to-landing-zone-workflow]
+    permissions:
+      contents: read
+      id-token: write
+    uses: "broadinstitute/dsp-reusable-workflows/.github/workflows/cromwell-az-e2e-test.yaml@main"
+    with:
+      branch: "${{ inputs.target-branch }}"
+      bee-name: "${{ needs.params-gen.outputs.bee-name }}"
+      billing-project-name: "${{ needs.params-gen.outputs.project-name }}"
+
+  delete-billing-project-v2-from-bee-workflow:
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    needs: [run-cromwell-az-e2e, create-and-attach-billing-project-to-landing-zone-workflow, params-gen]
+    if: always()
+    steps:
+      - name: dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: .github/workflows/delete-billing-project-v2-from-bee.yaml
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.BROADBOT_TOKEN }}
+          inputs: '{ 
+            "run-name": "delete-billing-project-v2-from-bee-${{ env.RUN_NAME_SUFFIX }}",
+            "bee-name": "${{ needs.params-gen.outputs.bee-name }}", 
+            "billing-project": "${{ needs.params-gen.outputs.project-name }}", 
+            "billing-project-owner": "${{ inputs.owner-subject }}", 
+            "service-account": "${{ inputs.service-account }}", 
+            "silent-on-failure": "false" }'
+
+  destroy-bee-workflow:
+    runs-on: ubuntu-latest
+    needs: [params-gen, create-bee-workflow, delete-billing-project-v2-from-bee-workflow]
+    if: always()
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - name: dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: bee-destroy.yaml
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.BROADBOT_TOKEN }}
+          inputs: '{ "bee-name": "${{ needs.params-gen.outputs.bee-name }}" }'
+          wait-for-completion: true
+
+  report-workflow:
+    uses: broadinstitute/sherlock/.github/workflows/client-report-workflow.yaml@main
+    with:
+      notify-slack-channels-upon-workflow-failure: "#cromwell_jenkins_ci_errors"
+    permissions:
+      id-token: write

.gitignore

@@ -23,6 +23,9 @@ console_output.txt
 expected.json
 run_mode_metadata.json
 
+#bloop files
+/.bloop
+
 # custom config
 cromwell-executions
 cromwell-test-executions
@@ -55,3 +58,6 @@ tesk_application.conf
 **/venv/
 exome_germline_single_sample_v1.3/
 **/*.pyc
+
+# GHA credentials
+gha-creds-*.json