Pr coverage #3
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Define workflow name. | ||
| name: MatrixOne Utils CI | ||
| on: | ||
| workflow_call: | ||
| secrets: | ||
| TOKEN_ACTION: | ||
| description: "A token passed from the caller workflow" | ||
| required: true | ||
| S3ENDPOINT: | ||
| description: "S3ENDPOINT For Test" | ||
| required: true | ||
| S3REGION: | ||
| description: "S3REGION For Test" | ||
| required: true | ||
| S3APIKEY: | ||
| description: "S3APIKEY For Test" | ||
| required: true | ||
| S3APISECRET: | ||
| description: "S3APISECRET For Test" | ||
| required: true | ||
| S3BUCKET: | ||
| description: "S3BUCKET For Test" | ||
| required: true | ||
| DOCU_GROUP_HOOK: | ||
| description: "DOCU_GROUP_HOOK For Notice" | ||
| required: true | ||
| OPENAI_API_KEY: | ||
| description: "OPENAI_API_KEY For get PR Suggestion" | ||
| required: true | ||
| jobs: | ||
| check_organization_user: | ||
| runs-on: ubuntu-22.04 | ||
| outputs: | ||
| in_org: ${{ steps.check_in_org.outputs.in_org }} | ||
| safe_label: ${{ steps.check_safe_label.outputs.safe_label }} | ||
| steps: | ||
| - id: check_in_org | ||
| name: CHECK ORGANIZATION USER | ||
| run: | | ||
| PAGE=1; | ||
| PER_PAGE=100; | ||
| USER_IN_ORG="0"; | ||
| while true; do | ||
| users=$(curl -L \ | ||
| -H "Accept: application/vnd.github+json" \ | ||
| -H "Authorization: Bearer ${{ secrets.TOKEN_ACTION }}" \ | ||
| -H "X-GitHub-Api-Version: 2022-11-28" \ | ||
| "https://api.github.com/orgs/matrixorigin/members?page=$PAGE&per_page=$PER_PAGE"); | ||
| if [ $(echo $users | jq ".[].login" | grep -c "${{ github.event.pull_request.user.login }}" ) -eq 1 ]; then | ||
| USER_IN_ORG="1"; | ||
| break; | ||
| fi | ||
| if [ $(echo $users | jq ".[].login" | wc -l ) -eq $PER_PAGE ]; then | ||
| PAGE=$(($PAGE+1)); | ||
| users=""; | ||
| else | ||
| break; | ||
| fi | ||
| done | ||
| echo "in_org=$USER_IN_ORG" >> $GITHUB_OUTPUT; | ||
| - id: check_safe_label | ||
| name: CHECK PULL REQUEST LABEL | ||
| run: | | ||
| labels=$(curl -L \ | ||
| -H "Accept: application/vnd.github+json" \ | ||
| -H "Authorization: Bearer ${{ secrets.TOKEN_ACTION }}" \ | ||
| -H "X-GitHub-Api-Version: 2022-11-28" \ | ||
| '${{ github.event.pull_request.url}}' | jq ".labels"); | ||
| if [ $(echo $labels | jq ".[].name" | grep -c "safe-to-test" ) -ge 1 ]; then | ||
| echo "safe_label=1" >> $GITHUB_OUTPUT; | ||
| else | ||
| echo "safe_label=0" >> $GITHUB_OUTPUT; | ||
| fi | ||
| ut_coverage: | ||
| if: ${{ needs.check_organization_user.outputs.safe_label == '1' || needs.check_organization_user.outputs.in_org == '1' }} | ||
| name: PR Unit Test Coverage | ||
| needs: [check_organization_user] | ||
| runs-on: ubuntu-22.04 | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| fetch-depth: "1" | ||
| path: ./matrixone | ||
| repository: ${{ github.event.pull_request.head.repo.full_name }} | ||
| ref: ${{ github.event.pull_request.head.ref }} | ||
| - name: Generate diff.patch | ||
| run: | | ||
| cd $GITHUB_WORKSPACE/matrixone | ||
| git remote add upstream https://github.com/matrixorigin/matrixone.git | ||
| target_branch="${{ github.event.pull_request.base.ref }}" | ||
| current_branch="${{ github.event.pull_request.head.ref }}" | ||
| echo "upstream target_branch: $target_branch" | ||
| echo "origin current_branch: $current_branch" | ||
| git fetch upstream $target_branch --depth=1 | ||
| git fetch origin $current_branch --depth=1 | ||
| git diff upstream/$target_branch origin/$current_branch > diff.patch | ||
| - name: Set up Go and JAVA | ||
| uses: matrixorigin/CI/actions/setup-env@main | ||
| with: | ||
| setup-java: true | ||
| - name: Set Variables | ||
| run: | | ||
| echo "ut_report='UT-Report.out'" >> $GITHUB_ENV | ||
| echo "raw_ut_coverage='ut_coverage.raw'" >> $GITHUB_ENV | ||
| echo "raw_bvt_coverage='bvt_coverage.raw'" >> $GITHUB_ENV | ||
| echo "ut_coverage='ut_coverage.out'" >> $GITHUB_ENV | ||
| echo "bvt_coverage='bvt_coverage.out'" >> $GITHUB_ENV | ||
| echo "ut_html_coverage='ut_coverage.html'" >> $GITHUB_ENV | ||
| echo "bvt_html_coverage='bvt_coverage.html'" >> $GITHUB_ENV | ||
| echo "ut_excluded_pkg='pkg/pb\|pkg/sql/parsers/goyacc\|yaccpar'" >> $GITHUB_ENV | ||
| echo "bvt_excluded_pkg='pkg/pb\|yaccpar'" >> $GITHUB_ENV | ||
| - name: Clone test-tool repository | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| repository: matrixorigin/mo-tester | ||
| path: ./mo-tester | ||
| ref: main | ||
| - name: Start Unit Test | ||
| timeout-minutes: 120 | ||
| run: | | ||
| cd $GITHUB_WORKSPACE/matrixone | ||
| test_scope=$(go list ./... | grep -v 'driver\|engine/aoe\|engine/memEngine\|pkg/catalog') | ||
| make clean && make config && make cgo | ||
| echo "ut start" | ||
| CGO_CFLAGS="-I${GITHUB_WORKSPACE}/matrixone/cgo" CGO_LDFLAGS="-L${GITHUB_WORKSPACE}/matrixone/cgo -lmo" go test -v -tags matrixone_test -p 6 -covermode=count -coverprofile=${{ env.raw_ut_coverage }} -coverpkg=./pkg/... ${test_scope} | tee ${{ env.ut_report }} | ||
| echo "ut finished" | ||
| - name: Start BVT Test Service | ||
| if: ${{ always() && !cancelled() }} | ||
| timeout-minutes: 10 | ||
| run: | | ||
| cd $GITHUB_WORKSPACE/matrixone | ||
| sed -i.bak 's/level = "info"/level = "error"/g' ./etc/launch/log.toml | ||
| if [ -d "./store" ]; then rm -rf ./store; fi | ||
| GOLDFLAGS=-ldflags="-X 'main.GoVersion=$(go version)' -X 'main.BranchName=$(git rev-parse --abbrev-ref HEAD)' -X 'main.CommitID=$(git rev-parse HEAD)' -X 'main.BuildTime=$(date)' -X 'main.Version=$(git describe --always --tags $(git rev-list --tags --max-count=1))'" | ||
| export GOCOVERDIR=$GITHUB_WORKSPACE/matrixone | ||
| go build -cover "$GOLDFLAGS" -o mo-service ./cmd/mo-service/ | ||
| ./optools/run_bvt.sh $GITHUB_WORKSPACE/matrixone launch | ||
| - name: Start BVT Test | ||
| if: ${{ always() && !cancelled() }} | ||
| id: bvt_on_pr_version | ||
| run: | | ||
| export LC_ALL="C.UTF-8" | ||
| locale | ||
| cd $GITHUB_WORKSPACE/mo-tester | ||
| sed -i 's/ port: [0-9]*/ port: 12345/g' mo.yml | ||
| cat mo.yml | ||
| echo "============================" | ||
| ./run.sh -n -g -o -p $GITHUB_WORKSPACE/matrixone/test/distributed/cases -e optimistic 2>&1 | ||
| - name: Generate UT and BVT Coverage File | ||
| if: ${{ always() && !cancelled() }} | ||
| timeout-minutes: 10 | ||
| run: | | ||
| cd $GITHUB_WORKSPACE/matrixone | ||
| cat ${{ env.raw_ut_coverage }} | grep -v ${{ env.ut_excluded_pkg }} > ${{ env.ut_coverage }} | ||
| echo "ut_coverage finished" | ||
| go tool cover -o ${{ env.ut_html_coverage }} -html=${{ env.ut_coverage }} | ||
| echo "ut_html_coverage finished" | ||
| rm -rf ${{ env.raw_ut_coverage }} | ||
| ps -aux | grep 'mo-service' | grep -v 'grep' | awk '{print $2}' | xargs kill -s SIGINT | ||
| i=1 && while [ -n "$(ps -aux|grep 'mo-service' | grep -v 'grep' | awk '{print $2}')" ] && [ $i -le 300 ]; do echo "mo-service kill not finished......$i/300"; i=$(($i+1)); sleep 1s; done | ||
| if [ $i -gt 300 ]; then echo 'stop mo-service failed...'; exit 1; else echo 'stop mo-service successed...'; fi | ||
| go tool covdata textfmt -i="$GITHUB_WORKSPACE/matrixone" -o ${{ env.raw_bvt_coverage }} | ||
| cat ${{ env.raw_bvt_coverage }} | grep -v ${{ env.bvt_excluded_pkg }} > ${{ env.bvt_coverage }} | ||
| echo "bvt_coverage finished" | ||
| go tool cover -o ${{ env.bvt_html_coverage }} -html=${{ env.bvt_coverage }} | ||
| echo "bvt_html_coverage finished" | ||
| - name: Download UT BVT Coverage and diff.patch Files | ||
| run: | | ||
| cd $GITHUB_WORKSPACE | ||
| mkdir -p coverage_downloads | ||
| cp $GITHUB_WORKSPACE/matrixone/ut_coverage.out $GITHUB_WORKSPACE/coverage_downloads/ut_coverage.out | ||
| cp $GITHUB_WORKSPACE/matrixone/bvt_coverage.out $GITHUB_WORKSPACE/coverage_downloads/bvt_coverage.out | ||
| cp $GITHUB_WORKSPACE/matrixone/diff.patch $GITHUB_WORKSPACE/coverage_downloads/diff.patch | ||
| - name: Compress Coverage Files | ||
| run: | | ||
| cd $GITHUB_WORKSPACE/coverage_downloads | ||
| sort ut_coverage.out | uniq -u > ut_coverage_sort.out | ||
| zip -r coverage_files.zip ut_coverage_sort.out bvt_coverage.out diff.patch | ||
| - name: Upload Compressed Files | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: coverage-files | ||
| path: ${{ github.workspace }}/coverage_downloads/coverage_files.zip | ||
| retention-days: 7 | ||
| - name: Set up Python environment | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: '3.9' # 使用Python 3.9版本 | ||
| - name: Checkout for Python file | ||
| uses: actions/checkout@v3 | ||
| with: | ||
| repository: matrixorigin/CI | ||
| fetch-depth: "0" | ||
| path: CI | ||
| - name: Get Python file | ||
| run: | | ||
| cd $GITHUB_WORKSPACE | ||
| cp $GITHUB_WORKSPACE/CI/scripts/parse_coverage.py $GITHUB_WORKSPACE/parse_coverage.py | ||
| - name: Run coverage Python script | ||
| run: | | ||
| cd $GITHUB_WORKSPACE | ||
| python $GITHUB_WORKSPACE/parse_coverage.py -coverage_files $GITHUB_WORKSPACE/matrixone/ut_coverage.out $GITHUB_WORKSPACE/matrixone/bvt_coverage.out -diff_path $GITHUB_WORKSPACE/matrixone/diff.patch | ||
| - name: Compress Python result file | ||
| run: | | ||
| cd $GITHUB_WORKSPACE | ||
| cp $GITHUB_WORKSPACE/merged_coverage.out $GITHUB_WORKSPACE/coverage_downloads/merged_coverage.out | ||
| cp $GITHUB_WORKSPACE/pr_coverage.out $GITHUB_WORKSPACE/coverage_downloads/pr_coverage.out | ||
| cd $GITHUB_WORKSPACE/coverage_downloads | ||
| zip -r python_result_files.zip merged_coverage.out pr_coverage.out | ||
| - name: Upload Python result file | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: python-result-files | ||
| path: ${{ github.workspace }}/coverage_downloads/python_result_files.zip | ||
| retention-days: 7 | ||
| - name: get code quality reports | ||
| if: ${{ always() && !cancelled() }} | ||
| run: | | ||
| cd $GITHUB_WORKSPACE/matrixone | ||
| # zip -r code_quality_reports.zip ${{ env.bvt_html_coverage }} ${{ env.ut_html_coverage }} coverage_report.txt UT-Report.out $GITHUB_WORKSPACE/matrixone/mo-service.log | ||
| zip -r code_quality_reports.zip ${{ env.bvt_html_coverage }} ${{ env.ut_html_coverage }} UT-Report.out $GITHUB_WORKSPACE/matrixone/mo-service.log | ||
| - uses: actions/upload-artifact@v4 | ||
| if: ${{ always() && !cancelled() }} | ||
| continue-on-error: true | ||
| with: | ||
| name: daily-code-quality-reports | ||
| path: | | ||
| ${{ github.workspace }}/matrixone/code_quality_reports.zip | ||
| retention-days: 7 | ||
| - uses: actions/upload-artifact@v4 | ||
| if: ${{ failure() }} | ||
| continue-on-error: true | ||
| with: | ||
| name: daily-test-reports | ||
| path: | | ||
| ${{ github.workspace }}/matrixone | ||
| retention-days: 7 | ||
| pr-size-label: | ||
| runs-on: ubuntu-22.04 | ||
| needs: [check_organization_user] | ||
| steps: | ||
| - name: size-label | ||
| uses: "matrixorigin/CI/actions/label-size-action@main" | ||
| with: | ||
| size_token: ${{ secrets.TOKEN_ACTION }} | ||
| ignore: ".md,.pb.go" | ||
| sizes: > | ||
| { | ||
| "XS":0, | ||
| "S":10, | ||
| "M":100, | ||
| "L":500, | ||
| "XL":1000, | ||
| "XXL":2000 | ||
| } | ||
| docu_issue_check: | ||
| runs-on: ubuntu-22.04 | ||
| needs: [check_organization_user] | ||
| if: ${{ github.event.action == 'opened' }} | ||
| steps: | ||
| - uses: matrixorigin/CI/actions/pull-issue-reviewer@main | ||
| with: | ||
| token_action: ${{ secrets.TOKEN_ACTION }} | ||
| uri_notice: ${{ secrets.DOCU_GROUP_HOOK }} | ||
| name_label: "doc influence" | ||
| reviewers: "lacrimosaprinz,dengn" | ||
| mentions: "Prinz,DengNan" | ||
| # chatgpt-review: | ||
| # name: ChatGPT Review | ||
| # needs: [check_organization_user] | ||
| # runs-on: ubuntu-22.04 | ||
| # if: ${{ github.event.action == 'opened' }} | ||
| # steps: | ||
| # - uses: matrixorigin/ChatGPT-Reviewer@main | ||
| # name: ChatGPT Review | ||
| # env: | ||
| # GITHUB_TOKEN: ${{ secrets.TOKEN_ACTION }} | ||
| # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | ||
| # with: | ||
| # model: "gpt-3.5-turbo" | ||
| # temperature: 0.2 | ||
| # review_per_file: false | ||
| # comment_per_file: false | ||
