From 8dd06ef704125a00956c04f09a646bc24ae77789 Mon Sep 17 00:00:00 2001 From: Karan Batavia Date: Mon, 9 Oct 2023 15:43:01 +0530 Subject: [PATCH 01/12] parallel processing for comparisom report --- .github/workflows/comparison-result.yml | 132 ++++++++++++++++++------ 1 file changed, 101 insertions(+), 31 deletions(-) diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml index 6fb35ad0..0aa8aa79 100644 --- a/.github/workflows/comparison-result.yml +++ b/.github/workflows/comparison-result.yml @@ -1,15 +1,38 @@ -name: Monitoring Stability and Comparing Results for privado +name: Monitoring Stability and Comparing Results -# Triggers when a pull_request or a push action is configured on master branch +# Triggers when a pull_request is created on: pull_request_target: + branches: + - "**" jobs: + start_workflow: + runs-on: ubuntu-latest + steps: + - name: Send message to slack + id: initial-message + uses: archive/github-actions-slack@master + with: + slack-optional-parse: full + slack-bot-user-oauth-access-token: ${{ secrets.SLACK_TOKEN }} + slack-channel: ${{ secrets.SLACK_CHANNEL_ID }} + slack-text: "Comparison workflow started for ${{github.event.pull_request.html_url}}" + + - name: Save output to env + id: save-output + run: echo "INIT_MSG_TS=${{ fromJson(steps.initial-message.outputs.slack-result).response.message.ts }}" >> $GITHUB_OUTPUT + outputs: + init_message_ts: ${{steps.save-output.outputs.INIT_MSG_TS}} setup_and_scan: + needs: start_workflow + strategy: + matrix: + language: ['java', 'python', 'js', 'ruby-1', 'ruby-2'] + continue-on-error: true runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Install JDK-18 uses: actions/setup-java@v3 with: @@ -26,53 +49,100 @@ jobs: with: python-version: '3.10' - - name: Clone standalone-monitoring-stability/main + - name: Clone standalone-monitoring-stability/flow-test uses: actions/checkout@v3 with: repository: Privado-Inc/standalone-monitoring-stability path: ./temp/standalone-monitoring-stability - ref: main - + ref: main + + # langauge specific repository file - name: Run the script for ${{github.head_ref}} and ${{github.base_ref}} - run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -brr ${{ github.event.pull_request.base.repo.html_url }} -hrr ${{ github.event.pull_request.head.repo.html_url }} -guf -urc + run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -r ./repos/${{matrix.language}}.txt -b ${{github.base_ref}} -h ${{github.head_ref}} -bcr ${{ github.event.pull_request.base.repo.html_url }} -hcr ${{ github.event.pull_request.head.repo.html_url }} -guf - name: Run aws-export - run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{github.event.number}} + run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{matrix.language}}-${{github.event.number}} - name: Move results to a folder - run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{github.event.number}}.xlsx ./results/output-${{github.event.number}}.xlsx && mv ./temp/result-${{github.event.number}}.zip ./results/result-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt + run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{matrix.language}}-${{github.event.number}}.xlsx ./results/output-${{matrix.language}}-${{github.event.number}}.xlsx && mv ./temp/result-${{matrix.language}}-${{github.event.number}}.zip ./results/result-${{matrix.language}}-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt + + # Send result to slack + - name: Zip the results + run: zip result-${{matrix.language}}-${{github.event.number}}.zip -r /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results + + - name: Set summary variable + run: | + echo "MESSAGE<> $GITHUB_ENV + echo "$(cat /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt)" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Post results to slack + uses: MeilCli/slack-upload-file@v3 + with: + thread_ts: ${{needs.start_workflow.outputs.init_message_ts}} + slack_token: ${{ secrets.SLACK_TOKEN }} + channel_id: ${{ secrets.SLACK_CHANNEL_ID }} + file_path: "/home/runner/work/privado-core/privado-core/result-${{matrix.language}}-${{github.event.number}}.zip" + initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado-core/pull/${{github.event.number}}\n Language: ${{matrix.language}} \nSummary Report:\n ${{ env.MESSAGE }}" + file_type: "zip" + + - name: Export workflow output + run: cd ./temp/standalone-monitoring-stability && python3 ./workflow_check.py /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt + + - name: Set summary variable + run: | + echo "MESSAGE<> $GITHUB_ENV + echo "$(cat ./temp/standalone-monitoring-stability/action_result.txt)" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV - - name: Upload output and result for next job + - name: Print action result + run: cat ./temp/standalone-monitoring-stability/action_result.txt + + - name: Upload summary file uses: actions/upload-artifact@master with: - name: results - path: /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results + name: ${{matrix.language}} + path: /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt - send-result: - needs: setup_and_scan - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 + - name: Workflow report analysis + if: ${{ env.MESSAGE != 'true' }} + run: exit 1 - - name: Download result folder + collate_summary: + needs: [start_workflow, setup_and_scan] + runs-on: ubuntu-latest + steps: + - name: Download summary file uses: actions/download-artifact@master with: - name: results - path: ./results - - - name: Zip the results - run: zip result-${{github.event.number}}.zip -r ./results + path: ./language_summary + + - name: Install Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Clone standalone-monitoring-stability/flow-test + uses: actions/checkout@v3 + with: + repository: Privado-Inc/standalone-monitoring-stability + path: ./temp/standalone-monitoring-stability + ref: main + + - name: Collate summary + run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./collate_summary.py -s /home/runner/work/privado-core/privado-core/language_summary - name: Set summary variable run: | echo "MESSAGE<> $GITHUB_ENV - echo "$(cat ./results/slack_summary.txt)" >> $GITHUB_ENV + echo "$(cat /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/global_summary.txt)" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - - name: Post results to slack - uses: MeilCli/slack-upload-file@v3 + + - name: Send summary to slack + uses: slackapi/slack-github-action@v1.24.0 with: - slack_token: ${{ secrets.SLACK_TOKEN }} - channel_id: ${{ secrets.SLACK_CHANNEL_ID }} - file_path: "/home/runner/work/privado/privado/result-${{github.event.number}}.zip" - initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}} \nSummary Report:\n ${{ env.MESSAGE }}" - file_type: "zip" \ No newline at end of file + update-ts: ${{needs.start_workflow.outputs.init_message_ts}} + channel-id: ${{ secrets.SLACK_CHANNEL_ID }} + slack-message: "\nComparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado-core/pull/${{github.event.number}}\nLanguage: All \nSummary Report:\n ${{ env.MESSAGE }}" + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }} From ebb7b87c5fa216ce59cc8b1151f90aa69be796ad Mon Sep 17 00:00:00 2001 From: Karan Batavia Date: Mon, 9 Oct 2023 15:52:52 +0530 Subject: [PATCH 02/12] dummy commit From 57478d001fd1bb6df8abf30334110ea45d8380d9 Mon Sep 17 00:00:00 2001 From: Karan Batavia Date: Mon, 9 Oct 2023 15:59:21 +0530 Subject: [PATCH 03/12] change command for standalone script --- .github/workflows/comparison-result.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml index 0aa8aa79..128313a3 100644 --- a/.github/workflows/comparison-result.yml +++ b/.github/workflows/comparison-result.yml @@ -58,7 +58,7 @@ jobs: # langauge specific repository file - name: Run the script for ${{github.head_ref}} and ${{github.base_ref}} - run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -r ./repos/${{matrix.language}}.txt -b ${{github.base_ref}} -h ${{github.head_ref}} -bcr ${{ github.event.pull_request.base.repo.html_url }} -hcr ${{ github.event.pull_request.head.repo.html_url }} -guf + run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -brr ${{ github.event.pull_request.base.repo.html_url }} -hrr ${{ github.event.pull_request.head.repo.html_url }} -guf -urc - name: Run aws-export run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{matrix.language}}-${{github.event.number}} @@ -146,3 +146,4 @@ jobs: slack-message: "\nComparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado-core/pull/${{github.event.number}}\nLanguage: All \nSummary Report:\n ${{ env.MESSAGE }}" env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }} + From e52d5207b57535f79748dee275b2b0089b18ff2d Mon Sep 17 00:00:00 2001 From: Karan Batavia Date: Mon, 9 Oct 2023 16:00:16 +0530 Subject: [PATCH 04/12] debug --- .github/workflows/comparison-result.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml index 128313a3..908d02eb 100644 --- a/.github/workflows/comparison-result.yml +++ b/.github/workflows/comparison-result.yml @@ -2,7 +2,7 @@ name: Monitoring Stability and Comparing Results # Triggers when a pull_request is created on: - pull_request_target: + pull_request: branches: - "**" From c50c165330a16a6607a9ee16acbddacf9eccf18a Mon Sep 17 00:00:00 2001 From: Karan Batavia Date: Mon, 9 Oct 2023 16:04:31 +0530 Subject: [PATCH 05/12] debug --- .github/workflows/comparison-result.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml index 908d02eb..71e1198f 100644 --- a/.github/workflows/comparison-result.yml +++ b/.github/workflows/comparison-result.yml @@ -68,12 +68,12 @@ jobs: # Send result to slack - name: Zip the results - run: zip result-${{matrix.language}}-${{github.event.number}}.zip -r /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results + run: zip result-${{matrix.language}}-${{github.event.number}}.zip -r /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results - name: Set summary variable run: | echo "MESSAGE<> $GITHUB_ENV - echo "$(cat /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt)" >> $GITHUB_ENV + echo "$(cat /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt)" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - name: Post results to slack @@ -82,12 +82,12 @@ jobs: thread_ts: ${{needs.start_workflow.outputs.init_message_ts}} slack_token: ${{ secrets.SLACK_TOKEN }} channel_id: ${{ secrets.SLACK_CHANNEL_ID }} - file_path: "/home/runner/work/privado-core/privado-core/result-${{matrix.language}}-${{github.event.number}}.zip" - initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado-core/pull/${{github.event.number}}\n Language: ${{matrix.language}} \nSummary Report:\n ${{ env.MESSAGE }}" + file_path: "/home/runner/work/privado/privado/result-${{matrix.language}}-${{github.event.number}}.zip" + initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}}\n Language: ${{matrix.language}} \nSummary Report:\n ${{ env.MESSAGE }}" file_type: "zip" - name: Export workflow output - run: cd ./temp/standalone-monitoring-stability && python3 ./workflow_check.py /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt + run: cd ./temp/standalone-monitoring-stability && python3 ./workflow_check.py /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt - name: Set summary variable run: | @@ -102,7 +102,7 @@ jobs: uses: actions/upload-artifact@master with: name: ${{matrix.language}} - path: /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt + path: /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt - name: Workflow report analysis if: ${{ env.MESSAGE != 'true' }} @@ -130,12 +130,12 @@ jobs: ref: main - name: Collate summary - run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./collate_summary.py -s /home/runner/work/privado-core/privado-core/language_summary + run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./collate_summary.py -s /home/runner/work/privado/privado/language_summary - name: Set summary variable run: | echo "MESSAGE<> $GITHUB_ENV - echo "$(cat /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/global_summary.txt)" >> $GITHUB_ENV + echo "$(cat /home/runner/work/privado/privado/temp/standalone-monitoring-stability/global_summary.txt)" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - name: Send summary to slack @@ -143,7 +143,7 @@ jobs: with: update-ts: ${{needs.start_workflow.outputs.init_message_ts}} channel-id: ${{ secrets.SLACK_CHANNEL_ID }} - slack-message: "\nComparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado-core/pull/${{github.event.number}}\nLanguage: All \nSummary Report:\n ${{ env.MESSAGE }}" + slack-message: "\nComparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}}\nLanguage: All \nSummary Report:\n ${{ env.MESSAGE }}" env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }} From a3f1aa8221e118f24b7376e9d158add2631fe9c0 Mon Sep 17 00:00:00 2001 From: Karan Batavia Date: Mon, 9 Oct 2023 16:13:35 +0530 Subject: [PATCH 06/12] debug --- .github/workflows/comparison-result.yml | 88 ++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 3 deletions(-) diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml index 71e1198f..6afdd4f3 100644 --- a/.github/workflows/comparison-result.yml +++ b/.github/workflows/comparison-result.yml @@ -1,8 +1,9 @@ + name: Monitoring Stability and Comparing Results # Triggers when a pull_request is created on: - pull_request: + pull_request_target: branches: - "**" @@ -130,12 +131,12 @@ jobs: ref: main - name: Collate summary - run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./collate_summary.py -s /home/runner/work/privado/privado/language_summary + run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./collate_summary.py -s ./language_summary - name: Set summary variable run: | echo "MESSAGE<> $GITHUB_ENV - echo "$(cat /home/runner/work/privado/privado/temp/standalone-monitoring-stability/global_summary.txt)" >> $GITHUB_ENV + echo "$(cat ./temp/standalone-monitoring-stability/global_summary.txt)" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - name: Send summary to slack @@ -147,3 +148,84 @@ jobs: env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }} + + + +name: Monitoring Stability and Comparing Results for privado + +# Triggers when a pull_request or a push action is configured on master branch +on: + pull_request_target: + +jobs: + setup_and_scan: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install JDK-18 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '18' + + - name: Export Java Home Path + run: export PATH=$JAVA_HOME/bin:$PATH + + - name: Install sbt + run: mkdir -p ~/bin && curl -Ls https://raw.githubusercontent.com/dwijnand/sbt-extras/master/sbt > ~/bin/sbt && chmod 0755 ~/bin/sbt + - name: Install Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Clone standalone-monitoring-stability/main + uses: actions/checkout@v3 + with: + repository: Privado-Inc/standalone-monitoring-stability + path: ./temp/standalone-monitoring-stability + ref: main + + - name: Run the script for ${{github.head_ref}} and ${{github.base_ref}} + run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -brr ${{ github.event.pull_request.base.repo.html_url }} -hrr ${{ github.event.pull_request.head.repo.html_url }} -guf -urc + + - name: Run aws-export + run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{github.event.number}} + + - name: Move results to a folder + run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{github.event.number}}.xlsx ./results/output-${{github.event.number}}.xlsx && mv ./temp/result-${{github.event.number}}.zip ./results/result-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt + + - name: Upload output and result for next job + uses: actions/upload-artifact@master + with: + name: results + path: /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results + + send-result: + needs: setup_and_scan + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Download result folder + uses: actions/download-artifact@master + with: + name: results + path: ./results + + - name: Zip the results + run: zip result-${{github.event.number}}.zip -r ./results + + - name: Set summary variable + run: | + echo "MESSAGE<> $GITHUB_ENV + echo "$(cat ./results/slack_summary.txt)" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + - name: Post results to slack + uses: MeilCli/slack-upload-file@v3 + with: + slack_token: ${{ secrets.SLACK_TOKEN }} + channel_id: ${{ secrets.SLACK_CHANNEL_ID }} + file_path: "/home/runner/work/privado/privado/result-${{github.event.number}}.zip" + initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}} \nSummary Report:\n ${{ env.MESSAGE }}" + file_type: "zip" \ No newline at end of file From 1c8fe8c349f6bb7c5f6e4443df608f453baddf8f Mon Sep 17 00:00:00 2001 From: Karan Batavia Date: Mon, 9 Oct 2023 16:14:21 +0530 Subject: [PATCH 07/12] debug --- .github/workflows/comparison-result.yml | 82 ------------------------- 1 file changed, 82 deletions(-) diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml index 6afdd4f3..4d205e03 100644 --- a/.github/workflows/comparison-result.yml +++ b/.github/workflows/comparison-result.yml @@ -147,85 +147,3 @@ jobs: slack-message: "\nComparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}}\nLanguage: All \nSummary Report:\n ${{ env.MESSAGE }}" env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }} - - - - -name: Monitoring Stability and Comparing Results for privado - -# Triggers when a pull_request or a push action is configured on master branch -on: - pull_request_target: - -jobs: - setup_and_scan: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install JDK-18 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: '18' - - - name: Export Java Home Path - run: export PATH=$JAVA_HOME/bin:$PATH - - - name: Install sbt - run: mkdir -p ~/bin && curl -Ls https://raw.githubusercontent.com/dwijnand/sbt-extras/master/sbt > ~/bin/sbt && chmod 0755 ~/bin/sbt - - name: Install Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Clone standalone-monitoring-stability/main - uses: actions/checkout@v3 - with: - repository: Privado-Inc/standalone-monitoring-stability - path: ./temp/standalone-monitoring-stability - ref: main - - - name: Run the script for ${{github.head_ref}} and ${{github.base_ref}} - run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -brr ${{ github.event.pull_request.base.repo.html_url }} -hrr ${{ github.event.pull_request.head.repo.html_url }} -guf -urc - - - name: Run aws-export - run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{github.event.number}} - - - name: Move results to a folder - run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{github.event.number}}.xlsx ./results/output-${{github.event.number}}.xlsx && mv ./temp/result-${{github.event.number}}.zip ./results/result-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt - - - name: Upload output and result for next job - uses: actions/upload-artifact@master - with: - name: results - path: /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results - - send-result: - needs: setup_and_scan - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Download result folder - uses: actions/download-artifact@master - with: - name: results - path: ./results - - - name: Zip the results - run: zip result-${{github.event.number}}.zip -r ./results - - - name: Set summary variable - run: | - echo "MESSAGE<> $GITHUB_ENV - echo "$(cat ./results/slack_summary.txt)" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - name: Post results to slack - uses: MeilCli/slack-upload-file@v3 - with: - slack_token: ${{ secrets.SLACK_TOKEN }} - channel_id: ${{ secrets.SLACK_CHANNEL_ID }} - file_path: "/home/runner/work/privado/privado/result-${{github.event.number}}.zip" - initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}} \nSummary Report:\n ${{ env.MESSAGE }}" - file_type: "zip" \ No newline at end of file From 9cde320a6353d493ac61612b5d723b09e6bb198c Mon Sep 17 00:00:00 2001 From: Karan Batavia Date: Mon, 9 Oct 2023 16:18:43 +0530 Subject: [PATCH 08/12] debug --- .github/workflows/comparison-result.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml index 4d205e03..e8b3e9d0 100644 --- a/.github/workflows/comparison-result.yml +++ b/.github/workflows/comparison-result.yml @@ -34,6 +34,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Install JDK-18 uses: actions/setup-java@v3 with: @@ -50,23 +51,22 @@ jobs: with: python-version: '3.10' - - name: Clone standalone-monitoring-stability/flow-test + - name: Clone standalone-monitoring-stability/main uses: actions/checkout@v3 with: repository: Privado-Inc/standalone-monitoring-stability path: ./temp/standalone-monitoring-stability - ref: main - - # langauge specific repository file + ref: main + - name: Run the script for ${{github.head_ref}} and ${{github.base_ref}} run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -brr ${{ github.event.pull_request.base.repo.html_url }} -hrr ${{ github.event.pull_request.head.repo.html_url }} -guf -urc - name: Run aws-export - run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{matrix.language}}-${{github.event.number}} + run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{github.event.number}} - name: Move results to a folder - run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{matrix.language}}-${{github.event.number}}.xlsx ./results/output-${{matrix.language}}-${{github.event.number}}.xlsx && mv ./temp/result-${{matrix.language}}-${{github.event.number}}.zip ./results/result-${{matrix.language}}-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt - + run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{github.event.number}}.xlsx ./results/output-${{github.event.number}}.xlsx && mv ./temp/result-${{github.event.number}}.zip ./results/result-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt + # Send result to slack - name: Zip the results run: zip result-${{matrix.language}}-${{github.event.number}}.zip -r /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results @@ -146,4 +146,4 @@ jobs: channel-id: ${{ secrets.SLACK_CHANNEL_ID }} slack-message: "\nComparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}}\nLanguage: All \nSummary Report:\n ${{ env.MESSAGE }}" env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }} \ No newline at end of file From 509f24eba327dff0b6db32a3a472eda57bd95c9c Mon Sep 17 00:00:00 2001 From: Karan Batavia Date: Mon, 9 Oct 2023 16:33:01 +0530 Subject: [PATCH 09/12] debug --- .github/workflows/comparison-result.yml | 27 +++++++++++-------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml index e8b3e9d0..da6cee0a 100644 --- a/.github/workflows/comparison-result.yml +++ b/.github/workflows/comparison-result.yml @@ -1,7 +1,6 @@ +name: Monitoring Stability and Comparing Results for privado -name: Monitoring Stability and Comparing Results - -# Triggers when a pull_request is created +# Triggers when a pull_request or a push action is configured on master branch on: pull_request_target: branches: @@ -25,6 +24,7 @@ jobs: run: echo "INIT_MSG_TS=${{ fromJson(steps.initial-message.outputs.slack-result).response.message.ts }}" >> $GITHUB_OUTPUT outputs: init_message_ts: ${{steps.save-output.outputs.INIT_MSG_TS}} + setup_and_scan: needs: start_workflow strategy: @@ -59,15 +59,14 @@ jobs: ref: main - name: Run the script for ${{github.head_ref}} and ${{github.base_ref}} - run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -brr ${{ github.event.pull_request.base.repo.html_url }} -hrr ${{ github.event.pull_request.head.repo.html_url }} -guf -urc + run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -r ./repos/${{matrix.language}}.txt -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -brr ${{ github.event.pull_request.base.repo.html_url }} -hrr ${{ github.event.pull_request.head.repo.html_url }} -guf -urc - name: Run aws-export - run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{github.event.number}} + run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{matrix.language}}-${{github.event.number}} - name: Move results to a folder - run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{github.event.number}}.xlsx ./results/output-${{github.event.number}}.xlsx && mv ./temp/result-${{github.event.number}}.zip ./results/result-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt - - # Send result to slack + run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{matrix.language}}-${{github.event.number}}.xlsx ./results/output-${{matrix.language}}-${{github.event.number}}.xlsx && mv ./temp/result-${{matrix.language}}-${{github.event.number}}.zip ./results/result-${{matrix.language}}-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt + - name: Zip the results run: zip result-${{matrix.language}}-${{github.event.number}}.zip -r /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results @@ -89,16 +88,13 @@ jobs: - name: Export workflow output run: cd ./temp/standalone-monitoring-stability && python3 ./workflow_check.py /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt - + - name: Set summary variable run: | echo "MESSAGE<> $GITHUB_ENV echo "$(cat ./temp/standalone-monitoring-stability/action_result.txt)" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - - name: Print action result - run: cat ./temp/standalone-monitoring-stability/action_result.txt - - name: Upload summary file uses: actions/upload-artifact@master with: @@ -130,13 +126,14 @@ jobs: path: ./temp/standalone-monitoring-stability ref: main - - name: Collate summary - run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./collate_summary.py -s ./language_summary + - name: Collate summary + run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./collate_summary.py -s /home/runner/work/privado/privado/language_summary + - name: Set summary variable run: | echo "MESSAGE<> $GITHUB_ENV - echo "$(cat ./temp/standalone-monitoring-stability/global_summary.txt)" >> $GITHUB_ENV + echo "$(cat /home/runner/work/privado/privado/temp/standalone-monitoring-stability/global_summary.txt)" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - name: Send summary to slack From ade3bf99255d75b355e0e3cacf59f84a29444fe3 Mon Sep 17 00:00:00 2001 From: Karan Batavia Date: Mon, 9 Oct 2023 16:43:50 +0530 Subject: [PATCH 10/12] debug --- .github/workflows/comparison-result.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml index da6cee0a..6c557cbb 100644 --- a/.github/workflows/comparison-result.yml +++ b/.github/workflows/comparison-result.yml @@ -2,7 +2,7 @@ name: Monitoring Stability and Comparing Results for privado # Triggers when a pull_request or a push action is configured on master branch on: - pull_request_target: + pull_request: branches: - "**" From 5044ec2bdfcc30106bab530df45cbeedaab68b37 Mon Sep 17 00:00:00 2001 From: Karan Batavia Date: Mon, 9 Oct 2023 16:47:59 +0530 Subject: [PATCH 11/12] debug --- .github/workflows/comparison-result.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml index 6c557cbb..da6cee0a 100644 --- a/.github/workflows/comparison-result.yml +++ b/.github/workflows/comparison-result.yml @@ -2,7 +2,7 @@ name: Monitoring Stability and Comparing Results for privado # Triggers when a pull_request or a push action is configured on master branch on: - pull_request: + pull_request_target: branches: - "**" From 00216c8880d8f2cf93ae8354bef76c8d7c40a8fd Mon Sep 17 00:00:00 2001 From: Karan Batavia Date: Mon, 9 Oct 2023 16:56:30 +0530 Subject: [PATCH 12/12] debug --- .github/workflows/comparison-result.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml index da6cee0a..e46c7299 100644 --- a/.github/workflows/comparison-result.yml +++ b/.github/workflows/comparison-result.yml @@ -1,6 +1,7 @@ name: Monitoring Stability and Comparing Results for privado # Triggers when a pull_request or a push action is configured on master branch +# Run on all branches on: pull_request_target: branches: