Merge branch 'main' into rm-labels

.github/workflows/merge-trigger-tke.yaml

@@ -153,7 +153,7 @@ jobs:
         run: |
           export MYSQL_PWD=111;
           for i in {1..300}; do
-            if mysql -h ${{ steps.get_mo_addr.outputs.mo_svc_ip }} -P 6001 -u dump -e "select mo_ctl('cn', 'txn-trace', 'add-txn user dump');select mo_ctl('cn', 'txn-trace', 'add-txn user admin');select mo_ctl('cn', 'txn-trace', 'add-txn connection sql-executor,0');select mo_ctl('cn', 'txn-trace', 'enable txn');select mo_ctl('cn', 'txn-trace', 'enable data');"; then
+            if mysql -h ${{ steps.get_mo_addr.outputs.mo_svc_ip }} -P 6001 -u dump -e "select mo_ctl('cn', 'txn-trace', 'add-txn user dump');select mo_ctl('cn', 'txn-trace', 'add-txn user admin');select mo_ctl('cn', 'txn-trace', 'add-txn connection sql-executor,0');select mo_ctl('cn', 'txn-trace', 'enable txn');select mo_ctl('cn', 'txn-trace', 'enable data');select mo_ctl('cn', 'txn-trace', 'enable txn-action');select mo_ctl('cn', 'txn-trace', 'add-statement cost 10s');"; then
               break;
             fi
           sleep 1;
@@ -243,7 +243,7 @@ jobs:
           echo "$JAVA_HOME/bin" >> $GITHUB_PATH
       - name: Generate MO-Tester Config and Start BVT Test
         if: ${{ always() && !cancelled() }}
-        timeout-minutes: 30
+        timeout-minutes: 40
         run: |
           export LC_ALL="C.UTF-8"
           locale
@@ -560,7 +560,7 @@ jobs:
           done
  
       # start sysbench test
-      - name: Start OLTP Mixed Test With 100 Threads
+      - name: Start OLTP Mixed Test With 30 Threads
         if: ${{ always() && !cancelled() }}
         timeout-minutes: 40
         run: |
@@ -570,8 +570,8 @@ jobs:
           
           cd $GITHUB_WORKSPACE/mo-load
           mkdir -p report
-          echo 'Start OLTP Insert Test With 100 threads'
-          ./start.sh -c cases/sysbench/mixed_10_100000 -t 100 -d 20 -g -b sbtest | tee report/oltp-mix.log
+          echo 'Start OLTP Insert Test With 30 threads'
+          ./start.sh -c cases/sysbench/mixed_10_100000 -t 30 -d 20 -g -b sbtest | tee report/oltp-mix.log
      
       - name: Result Summary
         if: ${{ always() && !cancelled()  }}

.github/workflows/robot.yaml

@@ -301,4 +301,31 @@ jobs:
           github_token: ${{ secrets.TOKEN_ACTION }}  # 获取github action token操作权限
           labels: 'needs-triage'  #要删除的label标签 形如 label1,label2,...
           current_assignee: ${{ github.event.assignee.login }}
-          blacklist: 'matrix-meow'  #黑名单形如: user1,user2,...
+          blacklist: 'matrix-meow'  #黑名单形如: user1,user2,...
+
+  reopen-without-PR:
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'issues' && github.event.action == 'closed' && github.actor != 'sukki37' }}
+    steps:
+      - name: Reopen Issue Without PR
+        id: step1
+        env:
+          BASE_URL: https://api.github.com
+        with:
+          OWNER: ${{ github.repository_owner }} # 获取执行该作业的仓库作者 形如: matrixorigin
+          REPOSITORY: ${{ github.repository }}  # 获取执行该作业的仓库名 形如: matrixorigin/CI
+          ISSUE_NUMBER: ${{ github.event.issue.number }}  # 获取当前操作的issue number
+          GITHUB_TOKEN: ${{ secrets.TOKEN_ACTION }}  # 获取github action token操作权限
+          ASSIGNEES: sukki37  #issue指定分配对象
+          LABELS: 'no-pr-linked'  #相关自定义label配置 形如 label1,label2
+          LABELS_NEED: 'tech-request,feature,Feature,kind/feature,attention/feature-incomplete,bug/ut,Bug fix,kind/bug,kind/subtask,kind/tech-request'  # 指定需要排查的标签 label1,label2,...
+        uses: matrixorigin/CI/actions/reopen-without-PR@main
+      - name: WeChat Work notification
+        if: ${{ failure() || cancelled() || steps.step1.outputs.send == 'yes' }}
+        id: notification
+        uses: chf007/action-wechat-work@master
+        env:
+          WECHAT_WORK_BOT_WEBHOOK: ${{secrets.WECHAT_WORK_BOT_WEBHOOK}}
+        with:
+          msgtype: markdown
+          content: "MO Close Issue Without PR: [${{ github.event.issue.number }}](${{ github.event.issue.html_url }})\nCreator:<@${{ github.event.issue.user.login }}>"

.github/workflows/utils.yaml

@@ -75,134 +75,181 @@ jobs:
             echo "safe_label=0" >> $GITHUB_OUTPUT;
           fi
 
-  ut_coverage:
+  pr_coverage:
     if: ${{ needs.check_organization_user.outputs.safe_label == '1' || needs.check_organization_user.outputs.in_org == '1' }}
-    name: PR Unit Test Coverage
+    name: PR Unit and BVT Test Coverage
     needs: [check_organization_user]
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: "3"
+          fetch-depth: "1"
           path: ./matrixone
           repository: ${{ github.event.pull_request.head.repo.full_name }}
           ref: ${{ github.event.pull_request.head.ref }}
-      - name: Set up Go and Java
+      - name: Generate diff.patch
+        run: |
+          cd $GITHUB_WORKSPACE/matrixone
+          git remote add upstream https://github.com/matrixorigin/matrixone.git
+          target_branch="${{ github.event.pull_request.base.ref }}"
+          current_branch="${{ github.event.pull_request.head.ref }}"
+          echo "upstream target_branch: $target_branch"
+          echo "origin current_branch: $current_branch"
+          
+          git fetch upstream $target_branch --depth=1
+          git fetch origin $current_branch --depth=1
+          
+          git diff upstream/$target_branch origin/$current_branch > diff.patch
+
+      - name: Set up Go and JAVA
         uses: matrixorigin/CI/actions/setup-env@main
+        with:
+          setup-java: true
       - name: Set Variables
         run: |
-          cd $GITHUB_WORKSPACE/matrixone
-          echo "test_scope=$(go list ./... | grep -v 'driver\|engine/aoe\|engine/memEngine\|pkg/catalog' | xargs)" >> $GITHUB_ENV
+          echo "ut_report='UT-Report.out'" >> $GITHUB_ENV
           echo "raw_ut_coverage='ut_coverage.raw'" >> $GITHUB_ENV
           echo "raw_bvt_coverage='bvt_coverage.raw'" >> $GITHUB_ENV
-          echo "raw_bvt_coverage_temp='bvt_coverage_temp.raw'" >> $GITHUB_ENV
-          echo "raw_ut_coverage_temp='ut_coverage_temp.raw'" >> $GITHUB_ENV
-          echo "all_coverage='all_coverage.raw'" >> $GITHUB_ENV
-          echo "test_report='test_report.out'" >> $GITHUB_ENV
+          echo "ut_coverage='ut_coverage.out'" >> $GITHUB_ENV
+          echo "bvt_coverage='bvt_coverage.out'" >> $GITHUB_ENV
           echo "ut_html_coverage='ut_coverage.html'" >> $GITHUB_ENV
           echo "bvt_html_coverage='bvt_coverage.html'" >> $GITHUB_ENV
-          echo "all_html_coverage='all_coverage.html'" >> $GITHUB_ENV
           echo "ut_excluded_pkg='pkg/pb\|pkg/sql/parsers/goyacc\|yaccpar'" >> $GITHUB_ENV
           echo "bvt_excluded_pkg='pkg/pb\|yaccpar'" >> $GITHUB_ENV
-          echo "coverage_line=75" >> $GITHUB_ENV
-      - name: Set env
-        run: |
-          echo "endpoint=${{ secrets.S3ENDPOINT }}" >> $GITHUB_ENV
-          echo "region=${{ secrets.S3REGION }}" >> $GITHUB_ENV
-          echo "apikey=${{ secrets.S3APIKEY }}" >> $GITHUB_ENV
-          echo "apisecret=${{ secrets.S3APISECRET }}" >> $GITHUB_ENV
-          echo "bucket=${{ secrets.S3BUCKET }}" >> $GITHUB_ENV
-      - name: Get Changed Paths
-        uses: matrixorigin/CI/actions/change-paths-action@main
-        id: ut-pr
-        with:
-          github-token: ${{ secrets.TOKEN_ACTION }}
-          ignore: "/,etc/,optools/,LICENSES/,cgo/,cmd/,docs/,proto/,test/,.github/,pkg/pb/"
-          source-path: ${{ env.test_scope }}
-      - name: Start PR Unit Test
-        if: ${{ steps.ut-pr.outputs.paths != '' }}
-        id: test_ut
-        timeout-minutes: 60
-        run: |
-          cd $GITHUB_WORKSPACE/matrixone
-          echo "-----------------------ut start-----------------------"
-          CGO_CFLAGS="-I${GITHUB_WORKSPACE}/matrixone/cgo"
-          CGO_LDFLAGS="-L${GITHUB_WORKSPACE}/matrixone/cgo -lmo -lm"
-          make clean && make config && make cgo
-          go test -v -cover -short -tags matrixone_test -p 6 -covermode=set -coverprofile ${{ env.raw_ut_coverage }} -count=1 ${{ steps.ut-pr.outputs.paths }}
-          echo "-----------------------ut finish-----------------------"
-      - name: Clone mo-tester repository
-        if: ${{ steps.ut-pr.outputs.paths != '' }}
+
+      - name: Clone test-tool repository
         uses: actions/checkout@v4
         with:
           repository: matrixorigin/mo-tester
           path: ./mo-tester
           ref: main
+      - name: Start Unit Test
+        timeout-minutes: 120
+        run: |
+          cd $GITHUB_WORKSPACE/matrixone
+          test_scope=$(go list ./... | grep -v 'driver\|engine/aoe\|engine/memEngine\|pkg/catalog')
+          make clean && make config && make cgo
+          echo "ut start"
+          CGO_CFLAGS="-I${GITHUB_WORKSPACE}/matrixone/cgo" CGO_LDFLAGS="-L${GITHUB_WORKSPACE}/matrixone/cgo -lmo" go test -v -tags matrixone_test -p 6 -covermode=set -coverprofile=${{ env.raw_ut_coverage }} -coverpkg=./pkg/... ${test_scope} | tee ${{ env.ut_report }}
+          echo "ut finished"
       - name: Start BVT Test Service
-        if: ${{ steps.ut-pr.outputs.paths != '' }}
-        id: test_bvt_start
+        if: ${{ always() && !cancelled() }}
         timeout-minutes: 10
         run: |
           cd $GITHUB_WORKSPACE/matrixone
+          sed -i.bak 's/level = "info"/level = "error"/g'  ./etc/launch/log.toml
           if  [ -d  "./store"  ]; then rm -rf ./store; fi
           GOLDFLAGS=-ldflags="-X 'main.GoVersion=$(go version)' -X 'main.BranchName=$(git rev-parse --abbrev-ref HEAD)' -X 'main.CommitID=$(git rev-parse HEAD)' -X 'main.BuildTime=$(date)' -X 'main.Version=$(git describe --always --tags $(git rev-list --tags --max-count=1))'"
           export GOCOVERDIR=$GITHUB_WORKSPACE/matrixone
           go build -cover "$GOLDFLAGS" -o mo-service ./cmd/mo-service/
           ./optools/run_bvt.sh $GITHUB_WORKSPACE/matrixone launch
-      - name: Start PR BVT Test
-        if: ${{ steps.ut-pr.outputs.paths != '' }}
-        id: test_bvt
-        continue-on-error: true
+      - name: Start BVT Test
+        if: ${{ always() && !cancelled() }}
+        id: bvt_on_pr_version
         run: |
+          export LC_ALL="C.UTF-8"
+          locale
+
           cd $GITHUB_WORKSPACE/mo-tester
           sed -i 's/  port: [0-9]*/  port: 12345/g' mo.yml
           cat mo.yml
-          echo "============================="
+          echo "============================"
+
           ./run.sh -n -g -o -p $GITHUB_WORKSPACE/matrixone/test/distributed/cases -e optimistic 2>&1
-      - name: Stop MO Service
-        id: test_bvt_stop
-        if: ${{ steps.ut-pr.outputs.paths != '' }}
+
+
+      - name: Generate UT and BVT Coverage File
+        if: ${{ always() && !cancelled() }}
+        timeout-minutes: 10
         run: |
+          cd $GITHUB_WORKSPACE/matrixone
+          cat ${{ env.raw_ut_coverage }} | grep -v ${{ env.ut_excluded_pkg }} > ${{ env.ut_coverage }}
+          echo "ut_coverage finished"
+          go tool cover -o ${{ env.ut_html_coverage }} -html=${{ env.ut_coverage }}
+          echo "ut_html_coverage finished"
+          rm -rf ${{ env.raw_ut_coverage }}
           ps -aux | grep 'mo-service' | grep -v 'grep' | awk '{print $2}' | xargs kill -s SIGINT
           i=1 && while [ -n "$(ps -aux|grep 'mo-service' | grep -v 'grep' | awk '{print $2}')" ] && [ $i -le 300 ]; do echo "mo-service kill not finished......$i/300"; i=$(($i+1)); sleep 1s; done
           if [ $i -gt 300 ]; then echo 'stop mo-service failed...'; exit 1; else echo 'stop mo-service successed...'; fi
-      - name: Coverage Calculate
-        if: ${{ steps.ut-pr.outputs.paths != '' }}
-        id: calculate_result
-        run: |
-          cd $GITHUB_WORKSPACE/matrixone
           go tool covdata textfmt -i="$GITHUB_WORKSPACE/matrixone" -o ${{ env.raw_bvt_coverage }}
-          cat ${{ env.raw_bvt_coverage }} | grep "${{ steps.ut-pr.outputs.files }}" | grep -v ${{ env.bvt_excluded_pkg }} | tee ${{ env.raw_bvt_coverage_temp }} > /dev/null
-          cat ${{ env.raw_ut_coverage }} | grep "${{ steps.ut-pr.outputs.files }}" | grep -v ${{ env.ut_excluded_pkg }} | tee ${{ env.raw_ut_coverage_temp }} > /dev/null
-          echo "======================= Coverage Calculate Start ======================="
-          awk -F '[ ]+' 'FNR==NR{arr[$1,$2]=$3;next}{if($3>=1){arr[$1,$2]=1}else{arr[$1,$2]+=$3}}END{for(i in arr){split(i,ind,SUBSEP);printf("%s %s %s\n",ind[1],ind[2],arr[i])};}' ${{ env.raw_ut_coverage_temp }} ${{ env.raw_bvt_coverage_temp }} | tee ${{ env.all_coverage }} > /dev/null
-           awk -F '[:, ]+' '{all[$1]+=$4;hit[$1]+=0;if($5>0) hit[$1]+=$4;}END{for(i in all)if(all[i]!=0){print(i,all[i],hit[i],hit[i]/all[i]*100)}else{print(i,all[i],hit[i],100)}}' ${{ env.all_coverage }} | tee ${{ env.test_report }} > /dev/null
-          awk 'BEGIN{flag=1}{if($4<${{ env.coverage_line }}){flag=0;print("\033[31merror:\033[0m the code coverage of \033[31m"$1"\033[0m is \033[31m"$4"%\033[0m which is below ${{ env.coverage_line }}%");} else { print("\033[32minfo:\033[0m the code coverage of "$1" is "$4"% which is up to ${{ env.coverage_line }}%") }}END{if(flag==0) {exit 1;}}' ${{ env.test_report }}
-      - name: Package Test Result
-        if: ${{ steps.ut-pr.outputs.paths != '' && failure() && steps.test_ut.conclusion != 'failure' && steps.test_bvt_start.conclusion != 'failure' && steps.test_bvt_stop.conclusion != 'failure' }}
-        continue-on-error: true
+          cat ${{ env.raw_bvt_coverage }} | grep -v ${{ env.bvt_excluded_pkg }} > ${{ env.bvt_coverage }}
+          echo "bvt_coverage finished"
+          go tool cover -o ${{ env.bvt_html_coverage }} -html=${{ env.bvt_coverage }}
+          echo "bvt_html_coverage finished"
+
+      - name: Download UT BVT Coverage and diff.patch Files
+        run: |
+          cd $GITHUB_WORKSPACE
+          mkdir -p coverage_downloads
+          cp $GITHUB_WORKSPACE/matrixone/ut_coverage.out $GITHUB_WORKSPACE/coverage_downloads/ut_coverage.out
+          cp $GITHUB_WORKSPACE/matrixone/bvt_coverage.out $GITHUB_WORKSPACE/coverage_downloads/bvt_coverage.out
+          cp $GITHUB_WORKSPACE/matrixone/diff.patch $GITHUB_WORKSPACE/coverage_downloads/diff.patch
+          
+          cd $GITHUB_WORKSPACE/coverage_downloads
+          sort ut_coverage.out | uniq > ut_coverage_sort.out
+          zip -r coverage_files.zip ut_coverage_sort.out bvt_coverage.out diff.patch
+          zip -r ut_origin_files.zip ut_coverage.out
+
+      - name: Upload Compressed Files
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-files
+          path: ${{ github.workspace }}/coverage_downloads/coverage_files.zip
+          retention-days: 7
+      - name: Upload origin ut_coverage.out
+        uses: actions/upload-artifact@v4
+        with:
+          name: origin-ut-coverage-files
+          path: ${{ github.workspace }}/coverage_downloads/ut_origin_files.zip
+          retention-days: 7
+      - name: Set up Python environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'  # 使用Python 3.9版本
+      - name: Checkout for Python file
+        uses: actions/checkout@v3
+        with:
+          repository: matrixorigin/CI
+          fetch-depth: "0"
+          path: CI
+      - name: Get Python file
+        run: |
+          cd $GITHUB_WORKSPACE
+          cp $GITHUB_WORKSPACE/CI/scripts/parse_coverage.py $GITHUB_WORKSPACE/parse_coverage.py
+      - name: Run coverage Python script
+        run: |
+          cd $GITHUB_WORKSPACE
+          python $GITHUB_WORKSPACE/parse_coverage.py -coverage_files $GITHUB_WORKSPACE/matrixone/ut_coverage.out $GITHUB_WORKSPACE/matrixone/bvt_coverage.out -diff_path $GITHUB_WORKSPACE/matrixone/diff.patch
+
+      - name: Compress final result files
         run: |
           cd $GITHUB_WORKSPACE/matrixone
-          echo "======================= Coverage HTML Generate Start ======================="
-          sed -i '1s/^/mode: set/' ${{ env.raw_ut_coverage_temp }}
-          go tool cover -o ${{ env.ut_html_coverage }} -html=${{ env.raw_ut_coverage_temp }}
-          echo "HTML for UT test generated is finished"
-          sed -i '1s/^/mode: set/' ${{ env.raw_bvt_coverage_temp }}
-          go tool cover -o ${{ env.bvt_html_coverage }} -html=${{ env.raw_bvt_coverage_temp }}
-          echo "HTML for BVT test generated is finished"
-          sed -i '1s/^/mode: set/' ${{ env.all_coverage }}
-          go tool cover -o ${{ env.all_html_coverage }} -html=${{ env.all_coverage }}
-          echo "HTML for ALL test generated is finished"
-          echo "======================= Coverage HTML Generate Finished ======================="
-          zip -r code_coverage_reports_${{ github.event.pull_request.number}}_${{ github.run_id }}.zip ${{ env.all_html_coverage }} ${{ env.raw_ut_coverage }} ${{ env.raw_ut_coverage_temp }} ${{ env.test_report }} ${{ env.raw_bvt_coverage_temp }} ${{ env.raw_bvt_coverage }} ${{ env.all_coverage }} ${{ env.ut_html_coverage }} ${{ env.bvt_html_coverage }} 'mo-service.log'
-          rm -rf ${{ env.all_html_coverage }} ${{ env.raw_ut_coverage }} ${{ env.raw_ut_coverage_temp }} ${{ env.test_report }} ${{ env.raw_bvt_coverage_temp }} ${{ env.raw_bvt_coverage }} ${{ env.all_coverage }} ${{ env.ut_html_coverage }} ${{ env.bvt_html_coverage }} 'mo-service.log'
+          cp $GITHUB_WORKSPACE/pr_coverage.out $GITHUB_WORKSPACE/matrixone/pr_coverage.out
+          go tool cover -o pr_coverage.html -html=pr_coverage.out
+          
+          cd $GITHUB_WORKSPACE
+          cp $GITHUB_WORKSPACE/matrixone/pr_coverage.html $GITHUB_WORKSPACE/coverage_downloads/pr_coverage.html
+          cp $GITHUB_WORKSPACE/pr_coverage.out $GITHUB_WORKSPACE/coverage_downloads/pr_coverage.out
+          cp $GITHUB_WORKSPACE/merged_coverage.out $GITHUB_WORKSPACE/coverage_downloads/merged_coverage.out
+          
+          cd $GITHUB_WORKSPACE/coverage_downloads
+          zip -r final_result_files.zip pr_coverage.html pr_coverage.out merged_coverage.out
+      - name: Upload final result files
+        uses: actions/upload-artifact@v4
+        if: ${{ always() && !cancelled() }}
+        continue-on-error: true
+        with:
+          name: final-result-files
+          path: |
+            ${{ github.workspace }}/coverage_downloads/final_result_files.zip
+          retention-days: 7
       - uses: actions/upload-artifact@v4
-        if: ${{ steps.ut-pr.outputs.paths != '' && failure() && steps.test_ut.conclusion != 'failure' && steps.test_bvt_start.conclusion != 'failure' && steps.test_bvt_stop.conclusion != 'failure' }}
+        if: ${{ failure() }}
         continue-on-error: true
         with:
-          name: PR-Unit-Test-Coverage-reports-${{ github.event.pull_request.number}}-${{ github.run_id }}.zip
+          name: daily-test-reports
           path: |
-            ${{ github.workspace }}/matrixone/code_coverage_reports_${{ github.event.pull_request.number}}_${{ github.run_id }}.zip
+            ${{ github.workspace }}/matrixone
           retention-days: 7
 
   pr-size-label:

actions/reopen-without-PR/Dockerfile

@@ -0,0 +1,15 @@
+FROM golang:alpine3.19 AS builder
+WORKDIR /workspace
+
+COPY . .
+
+RUN apk add make && make clean && make build
+
+
+FROM alpine:3.19
+COPY --from=builder /workspace/action /action
+
+RUN apk --no-cache add ca-certificates && rm -rf /var/cache/apk/* \
+    && update-ca-certificates
+
+ENTRYPOINT ["/action"]