.github/workflows/utils.yaml

# Define workflow name.
name: MatrixOne Utils CI

on:
  workflow_call:
    secrets:
      TOKEN_ACTION:
        description: "A token passed from the caller workflow"
        required: true
      S3ENDPOINT:
        description: "S3ENDPOINT For Test"
        required: true
      S3REGION:
        description: "S3REGION For Test"
        required: true
      S3APIKEY:
        description: "S3APIKEY For Test"
        required: true
      S3APISECRET:
        description: "S3APISECRET For Test"
        required: true
      S3BUCKET:
        description: "S3BUCKET For Test"
        required: true
      DOCU_GROUP_HOOK:
        description: "DOCU_GROUP_HOOK For Notice"
        required: true
      OPENAI_API_KEY:
        description: "OPENAI_API_KEY For get PR Suggestion"
        required: true

jobs:
  check_organization_user:
    runs-on: ubuntu-22.04
    outputs:
      in_org: ${{ steps.check_in_org.outputs.in_org }}
      safe_label: ${{ steps.check_safe_label.outputs.safe_label }}
    steps:
      - id: check_in_org
        name: CHECK ORGANIZATION USER
        run: |
          PAGE=1;
          PER_PAGE=100;
          USER_IN_ORG="0";
          while true; do
            users=$(curl -L \
              -H "Accept: application/vnd.github+json" \
              -H "Authorization: Bearer ${{ secrets.TOKEN_ACTION }}" \
              -H "X-GitHub-Api-Version: 2022-11-28" \
              "https://api.github.com/orgs/matrixorigin/members?page=$PAGE&per_page=$PER_PAGE");
            if [ $(echo $users | jq ".[].login" | grep -c "${{ github.event.pull_request.user.login }}" ) -eq 1 ]; then
              USER_IN_ORG="1";
              break;
            fi
            if [ $(echo $users | jq ".[].login" | wc -l ) -eq $PER_PAGE ]; then
              PAGE=$(($PAGE+1));
              users="";
            else
              break;
            fi
          done
          echo "in_org=$USER_IN_ORG" >> $GITHUB_OUTPUT;

      - id: check_safe_label
        name: CHECK PULL REQUEST LABEL
        run: |
          labels=$(curl -L \
            -H "Accept: application/vnd.github+json" \
            -H "Authorization: Bearer ${{ secrets.TOKEN_ACTION }}" \
            -H "X-GitHub-Api-Version: 2022-11-28" \
            '${{ github.event.pull_request.url}}' | jq ".labels");
          if [ $(echo $labels | jq ".[].name" | grep -c "safe-to-test" ) -ge 1 ]; then
            echo "safe_label=1" >> $GITHUB_OUTPUT;
          else
            echo "safe_label=0" >> $GITHUB_OUTPUT;
          fi

  ut_coverage:
    if: ${{ needs.check_organization_user.outputs.safe_label == '1' || needs.check_organization_user.outputs.in_org == '1' }}
    name: PR Unit Test Coverage
    needs: [check_organization_user]
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: "1"
          path: ./matrixone
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.event.pull_request.head.ref }}
      - name: Generate diff.patch
        run: |
          cd $GITHUB_WORKSPACE/matrixone
          git remote add upstream https://github.com/matrixorigin/matrixone.git
          target_branch="${{ github.event.pull_request.base.ref }}"
          current_branch="${{ github.event.pull_request.head.ref }}"
          echo "upstream target_branch: $target_branch"
          echo "origin current_branch: $current_branch"
          
          git fetch upstream $target_branch --depth=1
          git fetch origin $current_branch --depth=1
          
          git diff upstream/$target_branch origin/$current_branch > diff.patch

      - name: Set up Go and JAVA
        uses: matrixorigin/CI/actions/setup-env@main
        with:
          setup-java: true
      - name: Set Variables
        run: |
          echo "ut_report='UT-Report.out'" >> $GITHUB_ENV
          echo "raw_ut_coverage='ut_coverage.raw'" >> $GITHUB_ENV
          echo "raw_bvt_coverage='bvt_coverage.raw'" >> $GITHUB_ENV
          echo "ut_coverage='ut_coverage.out'" >> $GITHUB_ENV
          echo "bvt_coverage='bvt_coverage.out'" >> $GITHUB_ENV
          echo "ut_html_coverage='ut_coverage.html'" >> $GITHUB_ENV
          echo "bvt_html_coverage='bvt_coverage.html'" >> $GITHUB_ENV
          echo "ut_excluded_pkg='pkg/pb\|pkg/sql/parsers/goyacc\|yaccpar'" >> $GITHUB_ENV
          echo "bvt_excluded_pkg='pkg/pb\|yaccpar'" >> $GITHUB_ENV

      - name: Clone test-tool repository
        uses: actions/checkout@v4
        with:
          repository: matrixorigin/mo-tester
          path: ./mo-tester
          ref: main
      - name: Start Unit Test
        timeout-minutes: 120
        run: |
          cd $GITHUB_WORKSPACE/matrixone
          test_scope=$(go list ./... | grep -v 'driver\|engine/aoe\|engine/memEngine\|pkg/catalog')
          make clean && make config && make cgo
          echo "ut start"
          CGO_CFLAGS="-I${GITHUB_WORKSPACE}/matrixone/cgo" CGO_LDFLAGS="-L${GITHUB_WORKSPACE}/matrixone/cgo -lmo" go test -v -tags matrixone_test -p 6 -covermode=count -coverprofile=${{ env.raw_ut_coverage }} -coverpkg=./pkg/... ${test_scope} | tee ${{ env.ut_report }}
          echo "ut finished"
      - name: Start BVT Test Service
        if: ${{ always() && !cancelled() }}
        timeout-minutes: 10
        run: |
          cd $GITHUB_WORKSPACE/matrixone
          sed -i.bak 's/level = "info"/level = "error"/g'  ./etc/launch/log.toml
          if  [ -d  "./store"  ]; then rm -rf ./store; fi
          GOLDFLAGS=-ldflags="-X 'main.GoVersion=$(go version)' -X 'main.BranchName=$(git rev-parse --abbrev-ref HEAD)' -X 'main.CommitID=$(git rev-parse HEAD)' -X 'main.BuildTime=$(date)' -X 'main.Version=$(git describe --always --tags $(git rev-list --tags --max-count=1))'"
          export GOCOVERDIR=$GITHUB_WORKSPACE/matrixone
          go build -cover "$GOLDFLAGS" -o mo-service ./cmd/mo-service/
          ./optools/run_bvt.sh $GITHUB_WORKSPACE/matrixone launch
      - name: Start BVT Test
        if: ${{ always() && !cancelled() }}
        id: bvt_on_pr_version
        run: |
          export LC_ALL="C.UTF-8"
          locale

          cd $GITHUB_WORKSPACE/mo-tester
          sed -i 's/  port: [0-9]*/  port: 12345/g' mo.yml
          cat mo.yml
          echo "============================"

          ./run.sh -n -g -o -p $GITHUB_WORKSPACE/matrixone/test/distributed/cases -e optimistic 2>&1


      - name: Generate UT and BVT Coverage File
        if: ${{ always() && !cancelled() }}
        timeout-minutes: 10
        run: |
          cd $GITHUB_WORKSPACE/matrixone
          cat ${{ env.raw_ut_coverage }} | grep -v ${{ env.ut_excluded_pkg }} > ${{ env.ut_coverage }}
          echo "ut_coverage finished"
          go tool cover -o ${{ env.ut_html_coverage }} -html=${{ env.ut_coverage }}
          echo "ut_html_coverage finished"
          rm -rf ${{ env.raw_ut_coverage }}
          ps -aux | grep 'mo-service' | grep -v 'grep' | awk '{print $2}' | xargs kill -s SIGINT
          i=1 && while [ -n "$(ps -aux|grep 'mo-service' | grep -v 'grep' | awk '{print $2}')" ] && [ $i -le 300 ]; do echo "mo-service kill not finished......$i/300"; i=$(($i+1)); sleep 1s; done
          if [ $i -gt 300 ]; then echo 'stop mo-service failed...'; exit 1; else echo 'stop mo-service successed...'; fi
          go tool covdata textfmt -i="$GITHUB_WORKSPACE/matrixone" -o ${{ env.raw_bvt_coverage }}
          cat ${{ env.raw_bvt_coverage }} | grep -v ${{ env.bvt_excluded_pkg }} > ${{ env.bvt_coverage }}
          echo "bvt_coverage finished"
          go tool cover -o ${{ env.bvt_html_coverage }} -html=${{ env.bvt_coverage }}
          echo "bvt_html_coverage finished"

      - name: Download UT BVT Coverage and diff.patch Files
        run: |
          cd $GITHUB_WORKSPACE
          mkdir -p coverage_downloads
          cp $GITHUB_WORKSPACE/matrixone/ut_coverage.out $GITHUB_WORKSPACE/coverage_downloads/ut_coverage.out
          cp $GITHUB_WORKSPACE/matrixone/bvt_coverage.out $GITHUB_WORKSPACE/coverage_downloads/bvt_coverage.out
          cp $GITHUB_WORKSPACE/matrixone/diff.patch $GITHUB_WORKSPACE/coverage_downloads/diff.patch
      - name: Compress Coverage Files
        run: |
          cd $GITHUB_WORKSPACE/coverage_downloads
          sort ut_coverage.out | uniq -u > ut_coverage_sort.out
          zip -r coverage_files.zip ut_coverage_sort.out bvt_coverage.out diff.patch
      - name: Upload Compressed Files
        uses: actions/upload-artifact@v4
        with:
          name: coverage-files
          path: ${{ github.workspace }}/coverage_downloads/coverage_files.zip
          retention-days: 7


      - name: Set up Python environment
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'  # 使用Python 3.9版本
      - name: Checkout for Python file
        uses: actions/checkout@v3
        with:
          repository: matrixorigin/CI
          fetch-depth: "0"
          path: CI
      - name: Get Python file
        run: |
          cd $GITHUB_WORKSPACE
          cp $GITHUB_WORKSPACE/CI/scripts/parse_coverage.py $GITHUB_WORKSPACE/parse_coverage.py
      - name: Run coverage Python script
        run: |
          cd $GITHUB_WORKSPACE
          python $GITHUB_WORKSPACE/parse_coverage.py -coverage_files $GITHUB_WORKSPACE/matrixone/ut_coverage.out $GITHUB_WORKSPACE/matrixone/bvt_coverage.out -diff_path $GITHUB_WORKSPACE/matrixone/diff.patch
      - name: Compress Python result file
        run: |
          cd $GITHUB_WORKSPACE
          cp $GITHUB_WORKSPACE/merged_coverage.out $GITHUB_WORKSPACE/coverage_downloads/merged_coverage.out
          cp $GITHUB_WORKSPACE/pr_coverage.out $GITHUB_WORKSPACE/coverage_downloads/pr_coverage.out
          cd $GITHUB_WORKSPACE/coverage_downloads
          zip -r python_result_files.zip merged_coverage.out pr_coverage.out
      - name: upload python result file
        uses: actions/upload-artifact@v4
        with:
          name: python-result-files
          path: ${{ github.workspace }}/coverage_downloads/python_result_files.zip
          retention-days: 7

      - name: get code quality reports
        if: ${{ always() && !cancelled() }}
        run: |
          cd $GITHUB_WORKSPACE/matrixone
#          zip -r code_quality_reports.zip ${{ env.bvt_html_coverage }} ${{ env.ut_html_coverage }} coverage_report.txt UT-Report.out $GITHUB_WORKSPACE/matrixone/mo-service.log
          zip -r code_quality_reports.zip ${{ env.bvt_html_coverage }} ${{ env.ut_html_coverage }} UT-Report.out $GITHUB_WORKSPACE/matrixone/mo-service.log

      - uses: actions/upload-artifact@v4
        if: ${{ always() && !cancelled() }}
        continue-on-error: true
        with:
          name: daily-code-quality-reports
          path: |
            ${{ github.workspace }}/matrixone/code_quality_reports.zip
          retention-days: 7
      - uses: actions/upload-artifact@v4
        if: ${{ failure() }}
        continue-on-error: true
        with:
          name: daily-test-reports
          path: |
            ${{ github.workspace }}/matrixone
          retention-days: 7

  pr-size-label:
    runs-on: ubuntu-22.04
    needs: [check_organization_user]
    steps:
      - name: size-label
        uses: "matrixorigin/CI/actions/label-size-action@main"
        with:
          size_token: ${{ secrets.TOKEN_ACTION }}
          ignore: ".md,.pb.go"
          sizes: >
            {
              "XS":0,
              "S":10,
              "M":100,
              "L":500,
              "XL":1000,
              "XXL":2000
            }

  docu_issue_check:
    runs-on: ubuntu-22.04
    needs: [check_organization_user]
    if: ${{ github.event.action == 'opened' }}
    steps:
      - uses: matrixorigin/CI/actions/pull-issue-reviewer@main
        with:
          token_action: ${{ secrets.TOKEN_ACTION }}
          uri_notice: ${{ secrets.DOCU_GROUP_HOOK }}
          name_label: "doc influence"
          reviewers: "lacrimosaprinz,dengn"
          mentions: "Prinz,DengNan"

  # chatgpt-review:
  #   name: ChatGPT Review
  #   needs: [check_organization_user]
  #   runs-on: ubuntu-22.04
  #   if: ${{ github.event.action == 'opened' }}
  #   steps:
  #     - uses: matrixorigin/ChatGPT-Reviewer@main
  #       name: ChatGPT Review
  #       env:
  #         GITHUB_TOKEN: ${{ secrets.TOKEN_ACTION }}
  #         OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  #       with:
  #         model: "gpt-3.5-turbo"
  #         temperature: 0.2
  #         review_per_file: false
  #         comment_per_file: false