From 0e1bb1f40aeffc236fada503045c761e04f8265c Mon Sep 17 00:00:00 2001
From: Karan Batavia <118820668+karan-batavia@users.noreply.github.com>
Date: Mon, 9 Oct 2023 15:45:27 +0530
Subject: [PATCH 1/3] parallel processing for comparisom report (#321)

---
 .github/workflows/comparison-result.yml | 132 ++++++++++++++++++------
 1 file changed, 101 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml
index 6fb35ad0..0aa8aa79 100644
--- a/.github/workflows/comparison-result.yml
+++ b/.github/workflows/comparison-result.yml
@@ -1,15 +1,38 @@
-name: Monitoring Stability and Comparing Results for privado 
+name: Monitoring Stability and Comparing Results
 
-# Triggers when a pull_request or a push action is configured on master branch
+# Triggers when a pull_request is created
 on:
   pull_request_target:
+    branches:
+      - "**"
 
 jobs:
+  start_workflow:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Send message to slack
+        id: initial-message
+        uses: archive/github-actions-slack@master
+        with:
+          slack-optional-parse: full
+          slack-bot-user-oauth-access-token: ${{ secrets.SLACK_TOKEN }}
+          slack-channel: ${{ secrets.SLACK_CHANNEL_ID }}
+          slack-text: "Comparison workflow started for ${{github.event.pull_request.html_url}}"
+
+      - name: Save output to env
+        id: save-output
+        run: echo "INIT_MSG_TS=${{ fromJson(steps.initial-message.outputs.slack-result).response.message.ts }}" >> $GITHUB_OUTPUT
+    outputs:
+      init_message_ts: ${{steps.save-output.outputs.INIT_MSG_TS}}
   setup_and_scan:
+    needs: start_workflow
+    strategy:
+      matrix:
+        language: ['java', 'python', 'js', 'ruby-1', 'ruby-2']
+    continue-on-error: true
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-          
       - name: Install JDK-18
         uses: actions/setup-java@v3
         with:
@@ -26,53 +49,100 @@ jobs:
         with:
           python-version: '3.10'
 
-      - name: Clone standalone-monitoring-stability/main
+      - name: Clone standalone-monitoring-stability/flow-test
         uses: actions/checkout@v3
         with:
           repository: Privado-Inc/standalone-monitoring-stability
           path: ./temp/standalone-monitoring-stability
-          ref: main 
-      
+          ref: main
+
+      # langauge specific repository file
       - name: Run the script for ${{github.head_ref}} and ${{github.base_ref}}
-        run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -brr ${{ github.event.pull_request.base.repo.html_url }} -hrr ${{ github.event.pull_request.head.repo.html_url }} -guf -urc
+        run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -r ./repos/${{matrix.language}}.txt -b ${{github.base_ref}} -h ${{github.head_ref}} -bcr ${{ github.event.pull_request.base.repo.html_url }} -hcr ${{ github.event.pull_request.head.repo.html_url }} -guf
 
       - name: Run aws-export
-        run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{github.event.number}}
+        run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{matrix.language}}-${{github.event.number}}
 
       - name: Move results to a folder
-        run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{github.event.number}}.xlsx ./results/output-${{github.event.number}}.xlsx && mv ./temp/result-${{github.event.number}}.zip ./results/result-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt
+        run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{matrix.language}}-${{github.event.number}}.xlsx ./results/output-${{matrix.language}}-${{github.event.number}}.xlsx && mv ./temp/result-${{matrix.language}}-${{github.event.number}}.zip ./results/result-${{matrix.language}}-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt
+
+      # Send result to slack
+      - name: Zip the results
+        run: zip result-${{matrix.language}}-${{github.event.number}}.zip -r /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results
+
+      - name: Set summary variable
+        run: |
+          echo "MESSAGE<<EOF" >> $GITHUB_ENV
+          echo "$(cat /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt)" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+      - name: Post results to slack
+        uses: MeilCli/slack-upload-file@v3
+        with:
+          thread_ts: ${{needs.start_workflow.outputs.init_message_ts}}
+          slack_token: ${{ secrets.SLACK_TOKEN }}
+          channel_id: ${{ secrets.SLACK_CHANNEL_ID }}
+          file_path: "/home/runner/work/privado-core/privado-core/result-${{matrix.language}}-${{github.event.number}}.zip"
+          initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado-core/pull/${{github.event.number}}\n Language: ${{matrix.language}} \nSummary Report:\n ${{ env.MESSAGE }}"
+          file_type: "zip"
+
+      - name: Export workflow output
+        run: cd ./temp/standalone-monitoring-stability && python3 ./workflow_check.py /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt
+
+      - name: Set summary variable
+        run: |
+          echo "MESSAGE<<EOF" >> $GITHUB_ENV
+          echo "$(cat ./temp/standalone-monitoring-stability/action_result.txt)" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV 
 
-      - name: Upload output and result for next job
+      - name: Print action result
+        run: cat ./temp/standalone-monitoring-stability/action_result.txt
+
+      - name: Upload summary file
         uses: actions/upload-artifact@master
         with:
-          name: results
-          path: /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results
+          name: ${{matrix.language}}
+          path: /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt
 
-  send-result:
-    needs: setup_and_scan
-    runs-on: ubuntu-latest
-    steps: 
-      - uses: actions/checkout@v3
+      - name: Workflow report analysis
+        if: ${{ env.MESSAGE != 'true' }}
+        run: exit 1
 
-      - name: Download result folder
+  collate_summary:
+    needs: [start_workflow, setup_and_scan]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download summary file
         uses: actions/download-artifact@master
         with:
-          name: results
-          path: ./results
-      
-      - name: Zip the results
-        run: zip result-${{github.event.number}}.zip -r ./results
+          path: ./language_summary
+
+      - name: Install Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Clone standalone-monitoring-stability/flow-test
+        uses: actions/checkout@v3
+        with:
+          repository: Privado-Inc/standalone-monitoring-stability
+          path: ./temp/standalone-monitoring-stability
+          ref: main
+
+      - name: Collate summary
+        run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./collate_summary.py -s /home/runner/work/privado-core/privado-core/language_summary
 
       - name: Set summary variable
         run: |
           echo "MESSAGE<<EOF" >> $GITHUB_ENV
-          echo "$(cat ./results/slack_summary.txt)" >> $GITHUB_ENV
+          echo "$(cat /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/global_summary.txt)" >> $GITHUB_ENV
           echo "EOF" >> $GITHUB_ENV
-      - name: Post results to slack
-        uses: MeilCli/slack-upload-file@v3
+
+      - name: Send summary to slack
+        uses: slackapi/slack-github-action@v1.24.0
         with:
-          slack_token: ${{ secrets.SLACK_TOKEN }}
-          channel_id: ${{ secrets.SLACK_CHANNEL_ID }}
-          file_path: "/home/runner/work/privado/privado/result-${{github.event.number}}.zip"
-          initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}} \nSummary Report:\n ${{ env.MESSAGE }}"
-          file_type: "zip"
\ No newline at end of file
+          update-ts: ${{needs.start_workflow.outputs.init_message_ts}}
+          channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
+          slack-message: "\nComparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado-core/pull/${{github.event.number}}\nLanguage: All \nSummary Report:\n ${{ env.MESSAGE }}"
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }}

From edcc2004268148c1beae27e3d27aec8112949a13 Mon Sep 17 00:00:00 2001
From: Karan Batavia <118820668+karan-batavia@users.noreply.github.com>
Date: Mon, 9 Oct 2023 19:05:06 +0530
Subject: [PATCH 2/3] parallel processing for comparison report (#324)

---
 .github/workflows/comparison-result.yml | 34 ++++++++++++-------------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml
index 0aa8aa79..76dce609 100644
--- a/.github/workflows/comparison-result.yml
+++ b/.github/workflows/comparison-result.yml
@@ -1,4 +1,4 @@
-name: Monitoring Stability and Comparing Results
+name: Monitoring Stability and Comparing Results for privado
 
 # Triggers when a pull_request is created
 on:
@@ -24,6 +24,7 @@ jobs:
         run: echo "INIT_MSG_TS=${{ fromJson(steps.initial-message.outputs.slack-result).response.message.ts }}" >> $GITHUB_OUTPUT
     outputs:
       init_message_ts: ${{steps.save-output.outputs.INIT_MSG_TS}}
+
   setup_and_scan:
     needs: start_workflow
     strategy:
@@ -33,6 +34,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+
       - name: Install JDK-18
         uses: actions/setup-java@v3
         with:
@@ -49,16 +51,15 @@ jobs:
         with:
           python-version: '3.10'
 
-      - name: Clone standalone-monitoring-stability/flow-test
+      - name: Clone standalone-monitoring-stability/main
         uses: actions/checkout@v3
         with:
           repository: Privado-Inc/standalone-monitoring-stability
           path: ./temp/standalone-monitoring-stability
           ref: main
 
-      # langauge specific repository file
       - name: Run the script for ${{github.head_ref}} and ${{github.base_ref}}
-        run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -r ./repos/${{matrix.language}}.txt -b ${{github.base_ref}} -h ${{github.head_ref}} -bcr ${{ github.event.pull_request.base.repo.html_url }} -hcr ${{ github.event.pull_request.head.repo.html_url }} -guf
+        run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -r ./repos/${{matrix.language}}.txt -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -brr ${{ github.event.pull_request.base.repo.html_url }} -hrr ${{ github.event.pull_request.head.repo.html_url }} -guf -urc
 
       - name: Run aws-export
         run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{matrix.language}}-${{github.event.number}}
@@ -66,14 +67,13 @@ jobs:
       - name: Move results to a folder
         run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{matrix.language}}-${{github.event.number}}.xlsx ./results/output-${{matrix.language}}-${{github.event.number}}.xlsx && mv ./temp/result-${{matrix.language}}-${{github.event.number}}.zip ./results/result-${{matrix.language}}-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt
 
-      # Send result to slack
       - name: Zip the results
-        run: zip result-${{matrix.language}}-${{github.event.number}}.zip -r /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results
+        run: zip result-${{matrix.language}}-${{github.event.number}}.zip -r /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results
 
       - name: Set summary variable
         run: |
           echo "MESSAGE<<EOF" >> $GITHUB_ENV
-          echo "$(cat /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt)" >> $GITHUB_ENV
+          echo "$(cat /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt)" >> $GITHUB_ENV
           echo "EOF" >> $GITHUB_ENV
 
       - name: Post results to slack
@@ -82,12 +82,12 @@ jobs:
           thread_ts: ${{needs.start_workflow.outputs.init_message_ts}}
           slack_token: ${{ secrets.SLACK_TOKEN }}
           channel_id: ${{ secrets.SLACK_CHANNEL_ID }}
-          file_path: "/home/runner/work/privado-core/privado-core/result-${{matrix.language}}-${{github.event.number}}.zip"
-          initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado-core/pull/${{github.event.number}}\n Language: ${{matrix.language}} \nSummary Report:\n ${{ env.MESSAGE }}"
+          file_path: "/home/runner/work/privado/privado/result-${{matrix.language}}-${{github.event.number}}.zip"
+          initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}}\n Language: ${{matrix.language}} \nSummary Report:\n ${{ env.MESSAGE }}"
           file_type: "zip"
 
       - name: Export workflow output
-        run: cd ./temp/standalone-monitoring-stability && python3 ./workflow_check.py /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt
+        run: cd ./temp/standalone-monitoring-stability && python3 ./workflow_check.py /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt
 
       - name: Set summary variable
         run: |
@@ -95,14 +95,11 @@ jobs:
           echo "$(cat ./temp/standalone-monitoring-stability/action_result.txt)" >> $GITHUB_ENV
           echo "EOF" >> $GITHUB_ENV 
 
-      - name: Print action result
-        run: cat ./temp/standalone-monitoring-stability/action_result.txt
-
       - name: Upload summary file
         uses: actions/upload-artifact@master
         with:
           name: ${{matrix.language}}
-          path: /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/results/slack_summary.txt
+          path: /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt
 
       - name: Workflow report analysis
         if: ${{ env.MESSAGE != 'true' }}
@@ -129,13 +126,14 @@ jobs:
           path: ./temp/standalone-monitoring-stability
           ref: main
 
+
       - name: Collate summary
-        run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./collate_summary.py -s /home/runner/work/privado-core/privado-core/language_summary
+        run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./collate_summary.py -s /home/runner/work/privado/privado/language_summary
 
       - name: Set summary variable
         run: |
           echo "MESSAGE<<EOF" >> $GITHUB_ENV
-          echo "$(cat /home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/global_summary.txt)" >> $GITHUB_ENV
+          echo "$(cat /home/runner/work/privado/privado/temp/standalone-monitoring-stability/global_summary.txt)" >> $GITHUB_ENV
           echo "EOF" >> $GITHUB_ENV
 
       - name: Send summary to slack
@@ -143,6 +141,6 @@ jobs:
         with:
           update-ts: ${{needs.start_workflow.outputs.init_message_ts}}
           channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
-          slack-message: "\nComparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado-core/pull/${{github.event.number}}\nLanguage: All \nSummary Report:\n ${{ env.MESSAGE }}"
+          slack-message: "\nComparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}}\nLanguage: All \nSummary Report:\n ${{ env.MESSAGE }}"
         env:
-          SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }}
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }}
\ No newline at end of file

From c541ffbc5eb82120301d2cb10a68597af422a4a7 Mon Sep 17 00:00:00 2001
From: Karan Batavia <118820668+karan-batavia@users.noreply.github.com>
Date: Tue, 10 Oct 2023 09:55:53 +0530
Subject: [PATCH 3/3] Parallel comparison report (#326)

* parallel processing for comparison report

* change standalone ref to parallel-test

* change zip command to get only one folder

* change ref to main
---
 .github/workflows/comparison-result.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml
index 76dce609..2e89d18f 100644
--- a/.github/workflows/comparison-result.yml
+++ b/.github/workflows/comparison-result.yml
@@ -68,7 +68,7 @@ jobs:
         run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{matrix.language}}-${{github.event.number}}.xlsx ./results/output-${{matrix.language}}-${{github.event.number}}.xlsx && mv ./temp/result-${{matrix.language}}-${{github.event.number}}.zip ./results/result-${{matrix.language}}-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt
 
       - name: Zip the results
-        run: zip result-${{matrix.language}}-${{github.event.number}}.zip -r /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results
+        run: cd /home/runner/work/privado/privado/temp/standalone-monitoring-stability && zip result-${{matrix.language}}-${{github.event.number}}.zip -r ./results
 
       - name: Set summary variable
         run: |
@@ -82,7 +82,7 @@ jobs:
           thread_ts: ${{needs.start_workflow.outputs.init_message_ts}}
           slack_token: ${{ secrets.SLACK_TOKEN }}
           channel_id: ${{ secrets.SLACK_CHANNEL_ID }}
-          file_path: "/home/runner/work/privado/privado/result-${{matrix.language}}-${{github.event.number}}.zip"
+          file_path: "/home/runner/work/privado/privado/temp/standalone-monitoring-stability/result-${{matrix.language}}-${{github.event.number}}.zip"
           initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}}\n Language: ${{matrix.language}} \nSummary Report:\n ${{ env.MESSAGE }}"
           file_type: "zip"