From 55de5802766f7d4e873b72426e1cafd1d4d5f65b Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 31 Oct 2023 21:17:28 -0400 Subject: [PATCH 1/2] Onboard jenkins prod docker images in github actions Signed-off-by: Peter Zhu --- .github/workflows/add-untriaged.yml | 19 ---- .github/workflows/backport.yml | 30 ------- .github/workflows/codeql-analysis.yml | 38 -------- .github/workflows/dco.yml | 18 ---- .github/workflows/delete_backport_branch.yml | 15 ---- .../draft-release-notes-workflow.yml | 23 ----- .../workflows/integ-tests-with-security.yml | 49 +++++++++- .github/workflows/link-checker.yml | 25 ------ .github/workflows/maven-publish.yml | 38 -------- .github/workflows/sql-pitest.yml | 42 --------- .../workflows/sql-test-and-build-workflow.yml | 88 ------------------ .github/workflows/sql-test-workflow.yml | 89 ------------------- 12 files changed, 47 insertions(+), 427 deletions(-) delete mode 100644 .github/workflows/add-untriaged.yml delete mode 100644 .github/workflows/backport.yml delete mode 100644 .github/workflows/codeql-analysis.yml delete mode 100644 .github/workflows/dco.yml delete mode 100644 .github/workflows/delete_backport_branch.yml delete mode 100644 .github/workflows/draft-release-notes-workflow.yml delete mode 100644 .github/workflows/link-checker.yml delete mode 100644 .github/workflows/maven-publish.yml delete mode 100644 .github/workflows/sql-pitest.yml delete mode 100644 .github/workflows/sql-test-and-build-workflow.yml delete mode 100644 .github/workflows/sql-test-workflow.yml diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml deleted file mode 100644 index 15b9a55651..0000000000 --- a/.github/workflows/add-untriaged.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Apply 'untriaged' label during issue lifecycle - -on: - issues: - types: [opened, reopened, transferred] - -jobs: - apply-label: - runs-on: ubuntu-latest - steps: - - uses: actions/github-script@v6 - with: - script: | - github.rest.issues.addLabels({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - labels: ['untriaged'] - }) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml deleted file mode 100644 index 6472a968d8..0000000000 --- a/.github/workflows/backport.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Backport -on: - pull_request_target: - types: - - closed - - labeled - -jobs: - backport: - if: github.event.pull_request.merged == true - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - name: Backport - steps: - - name: GitHub App token - id: github_app_token - uses: tibdex/github-app-token@v1.5.0 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - installation_id: 22958780 - - - name: Backport - uses: VachaShah/backport@v2.2.0 - with: - github_token: ${{ steps.github_app_token.outputs.token }} - head_template: backport/backport-<%= number %>-to-<%= base %> - failure_labels: backport-failed diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 13c4ef0f60..0000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: "CodeQL" - -on: - pull_request: - branches: - - 'main' - - '[1-9]+.[0-9x]+' - push: - branches-ignore: - - 'dependabot/**' - paths: - - '**/*.java' - - '.github/workflows/codeql-analysis.yml' - -jobs: - analyze: - name: CodeQL-Scan - runs-on: ubuntu-latest - permissions: - security-events: write - actions: read - strategy: - fail-fast: false - matrix: - language: [ 'java' ] - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml deleted file mode 100644 index cf30ea89dc..0000000000 --- a/.github/workflows/dco.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Developer Certificate of Origin Check - -on: [pull_request] - -jobs: - check: - runs-on: ubuntu-latest - - steps: - - name: Get PR Commits - id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@v1.1.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: DCO Check - uses: tim-actions/dco@v1.1.0 - with: - commits: ${{ steps.get-pr-commits.outputs.commits }} diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml deleted file mode 100644 index 387a124b8c..0000000000 --- a/.github/workflows/delete_backport_branch.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: Delete merged branch of the backport PRs -on: - pull_request: - types: - - closed - -jobs: - delete-branch: - runs-on: ubuntu-latest - if: startsWith(github.event.pull_request.head.ref,'backport/') - steps: - - name: Delete merged branch - uses: SvanBoxel/delete-merged-branch@main - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml deleted file mode 100644 index 858e663d6b..0000000000 --- a/.github/workflows/draft-release-notes-workflow.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Release Drafter - -on: - pull_request: - push: - branches: - - 'main' - - '[1-9]+.[0-9x]+' - -jobs: - update_release_draft: - name: Update draft release notes - runs-on: ubuntu-latest - steps: - # Drafts your next Release notes as Pull Requests are merged into "develop" - - name: Update draft release notes - uses: release-drafter/release-drafter@v5 - with: - config-name: draft-release-notes-config.yml - tag: (None) - version: 3.0.0.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/integ-tests-with-security.yml b/.github/workflows/integ-tests-with-security.yml index 0d54b8cfef..4ff9ff6faa 100644 --- a/.github/workflows/integ-tests-with-security.yml +++ b/.github/workflows/integ-tests-with-security.yml @@ -10,11 +10,56 @@ on: - '.github/workflows/integ-tests-with-security.yml' jobs: - security-it: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + security-it-linux: + needs: Get-CI-Image-Tag + strategy: + fail-fast: false + matrix: + java: [ 11, 17 ] + + runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + + steps: + - uses: actions/checkout@v3 + + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + + - name: Build with Gradle + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew integTestWithSecurity" + + - name: Upload test reports + if: ${{ always() }} + uses: actions/upload-artifact@v2 + continue-on-error: true + with: + name: test-reports-${{ matrix.os }}-${{ matrix.java }} + path: | + integ-test/build/reports/** + integ-test/build/testclusters/*/logs/* + integ-test/build/testclusters/*/config/* + + security-it-windows-macos: strategy: fail-fast: false matrix: - os: [ ubuntu-latest, windows-latest, macos-latest ] + os: [ windows-latest, macos-latest ] java: [ 11, 17 ] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/link-checker.yml b/.github/workflows/link-checker.yml deleted file mode 100644 index 42c8fd35c8..0000000000 --- a/.github/workflows/link-checker.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Link Checker - -on: - pull_request: - push: - branches: - - 'main' - - '[1-9]+.[0-9x]+' - -jobs: - linkchecker: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: lychee Link Checker - id: lychee - uses: lycheeverse/lychee-action@master - with: - args: --accept=200,403,429,999 "./**/*.html" "./**/*.md" "./**/*.txt" --exclude "https://aws.oss.sonatype.*|http://localhost.*|https://localhost|https://odfe-node1:9200/|https://community.tableau.com/docs/DOC-17978|.*family.zzz|opensearch*|.*@amazon.com|.*email.com|.*@github.com|http://timestamp.verisign.com/scripts/timstamp.dll" - env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - - name: Fail if there were link errors - run: exit ${{ steps.lychee.outputs.exit_code }} diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml deleted file mode 100644 index 8adf7ae52c..0000000000 --- a/.github/workflows/maven-publish.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Publish snapshots to maven - -on: - workflow_dispatch: - push: - branches: - - main - - 1.* - - 2.* - -jobs: - build-and-publish-snapshots: - strategy: - fail-fast: false - if: github.repository == 'opensearch-project/sql' - runs-on: ubuntu-latest - - permissions: - id-token: write - contents: write - - steps: - - uses: actions/setup-java@v3 - with: - distribution: temurin # Temurin is a distribution of adoptium - java-version: 11 - - uses: actions/checkout@v3 - - uses: aws-actions/configure-aws-credentials@v1.7.0 - with: - role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }} - aws-region: us-east-1 - - name: publish snapshots to maven - run: | - export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text) - export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text) - echo "::add-mask::$SONATYPE_USERNAME" - echo "::add-mask::$SONATYPE_PASSWORD" - ./gradlew publishPluginZipPublicationToSnapshotsRepository diff --git a/.github/workflows/sql-pitest.yml b/.github/workflows/sql-pitest.yml deleted file mode 100644 index fb23ff829f..0000000000 --- a/.github/workflows/sql-pitest.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: SQL Plugin PiTest - -on: - workflow_dispatch: - inputs: - name: - required: false - type: string - -run-name: - ${{ inputs.name == '' && format('{0} @ {1}', github.ref_name, github.sha) || inputs.name }} - -jobs: - build: - strategy: - matrix: - java: - - 11 - - 17 - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: ${{ matrix.java }} - - - name: PiTest with Gradle - run: | - ./gradlew --continue :core:pitest :opensearch:pitest - - - name: Upload test reports - if: always() - uses: actions/upload-artifact@v2 - with: - name: test-reports-${{ matrix.entry.java }} - path: | - core/build/reports/** - opensearch/build/reports/** diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml deleted file mode 100644 index fecfe7adc2..0000000000 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: SQL Java CI - -on: - pull_request: - push: - branches-ignore: - - 'dependabot/**' - paths: - - '**/*.java' - - '**/*.g4' - - '!sql-jdbc/**' - - '**gradle*' - - '**lombok*' - - 'integ-test/**' - - '**/*.jar' - - '**/*.pom' - - '.github/workflows/sql-test-and-build-workflow.yml' - -jobs: - build: - strategy: - # Run all jobs - fail-fast: false - matrix: - entry: - - { os: ubuntu-latest, java: 11 } - - { os: windows-latest, java: 11, os_build_args: -x doctest -PbuildPlatform=windows } - - { os: macos-latest, java: 11} - - { os: ubuntu-latest, java: 17 } - - { os: windows-latest, java: 17, os_build_args: -x doctest -PbuildPlatform=windows } - - { os: macos-latest, java: 17 } - runs-on: ${{ matrix.entry.os }} - - steps: - - uses: actions/checkout@v3 - - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: ${{ matrix.entry.java }} - - - name: Build with Gradle - run: ./gradlew --continue build ${{ matrix.entry.os_build_args }} - - - name: Run backward compatibility tests - if: ${{ matrix.entry.os == 'ubuntu-latest' }} - run: ./scripts/bwctest.sh - - - name: Create Artifact Path - run: | - mkdir -p opensearch-sql-builds - cp -r ./plugin/build/distributions/*.zip opensearch-sql-builds/ - - # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action - - name: Upload SQL Coverage Report - if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} - uses: codecov/codecov-action@v3 - continue-on-error: true - with: - flags: sql-engine - token: ${{ secrets.CODECOV_TOKEN }} - - - name: Upload Artifacts - uses: actions/upload-artifact@v2 - continue-on-error: true - with: - name: opensearch-sql-${{ matrix.entry.os }}-${{ matrix.entry.java }} - path: opensearch-sql-builds - - - name: Upload test reports - if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} - uses: actions/upload-artifact@v2 - continue-on-error: true - with: - name: test-reports-${{ matrix.entry.os }}-${{ matrix.entry.java }} - path: | - sql/build/reports/** - ppl/build/reports/** - core/build/reports/** - common/build/reports/** - opensearch/build/reports/** - integ-test/build/reports/** - protocol/build/reports/** - legacy/build/reports/** - plugin/build/reports/** - doctest/build/testclusters/docTestCluster-0/logs/* - integ-test/build/testclusters/*/logs/* diff --git a/.github/workflows/sql-test-workflow.yml b/.github/workflows/sql-test-workflow.yml deleted file mode 100644 index 9ca27dffaf..0000000000 --- a/.github/workflows/sql-test-workflow.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: SQL Plugin Tests - -on: - workflow_dispatch: - inputs: - name: - required: false - type: string - -run-name: - ${{ inputs.name == '' && format('{0} @ {1}', github.ref_name, github.sha) || inputs.name }} - -jobs: - build: - strategy: - matrix: - java: - - 11 - - 17 - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: ${{ matrix.java }} - - - name: Run tests - id: tests - run: | - # Spotless - ./gradlew :opensearch:spotlessCheck || echo "* Spotless failed for opensearch" > report.log - ./gradlew :sql:spotlessCheck || echo "* Spotless failed for sql" >> report.log - ./gradlew :ppl:spotlessCheck || echo "* Spotless failed for ppl" >> report.log - ./gradlew :core:spotlessCheck || echo "* Spotless failed for core" >> report.log - ./gradlew :common:spotlessCheck || echo "* Spotless failed for common" >> report.log - ./gradlew :legacy:spotlessCheck || echo "* Spotless failed for legacy" >> report.log - ./gradlew :protocol:spotlessCheck || echo "* Spotless failed for protocol" >> report.log - ./gradlew :opensearch-sql-plugin:spotlessCheck || echo "* Spotless failed for plugin" >> report.log - ./gradlew :integ-test:spotlessCheck || echo "* Spotless failed for integ-test" >> report.log - # Unit tests - ./gradlew :opensearch:test || echo "* Unit tests failed for opensearch" >> report.log - ./gradlew :ppl:test || echo "* Unit tests failed for sql" >> report.log - ./gradlew :sql:test || echo "* Unit tests failed for ppl" >> report.log - ./gradlew :core:test || echo "* Unit tests failed for core" >> report.log - ./gradlew :protocol:test || echo "* Unit tests failed for protocol" >> report.log - ./gradlew :opensearch-sql-plugin:test || echo "* Unit tests failed for plugin" >> report.log - ./gradlew :legacy:test || echo "* Unit tests failed for legacy" >> report.log - # jacoco - ./gradlew :opensearch:jacocoTestCoverageVerification || echo "* Jacoco failed for opensearch" >> report.log - ./gradlew :ppl:jacocoTestCoverageVerification || echo "* Jacoco failed for sql" >> report.log - ./gradlew :sql:jacocoTestCoverageVerification || echo "* Jacoco failed for ppl" >> report.log - ./gradlew :core:jacocoTestCoverageVerification || echo "* Jacoco failed for core" >> report.log - ./gradlew :protocol:jacocoTestCoverageVerification || echo "* Jacoco failed for protocol" >> report.log - ./gradlew :opensearch-sql-plugin:jacocoTestCoverageVerification || echo "* Jacoco failed for plugin" >> report.log - # Misc tests - ./gradlew :integ-test:integTest || echo "* Integration test failed" >> report.log - ./gradlew :doctest:doctest || echo "* Doctest failed" >> report.log - ./scripts/bwctest.sh || echo "* Backward compatibility test failed" >> report.log - - - name: Verify test results - run: | - if [[ -e report.log ]] - then - echo "## FAILED TESTS :facepalm::warning::bangbang:" >> $GITHUB_STEP_SUMMARY - cat report.log >> $GITHUB_STEP_SUMMARY - exit 1 - fi - - - name: Upload test reports - if: always() - uses: actions/upload-artifact@v2 - with: - name: test-reports-${{ matrix.entry.java }} - path: | - sql/build/reports/** - ppl/build/reports/** - core/build/reports/** - common/build/reports/** - opensearch/build/reports/** - integ-test/build/reports/** - protocol/build/reports/** - legacy/build/reports/** - plugin/build/reports/** - doctest/build/testclusters/docTestCluster-0/logs/* - integ-test/build/testclusters/*/logs/* From 6b2f8655da27b95b4c4078a7f16ac30667f25d76 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 1 Nov 2023 13:18:44 -0400 Subject: [PATCH 2/2] Add more Signed-off-by: Peter Zhu --- .github/workflows/add-untriaged.yml | 19 ++ .github/workflows/backport.yml | 30 ++++ .github/workflows/codeql-analysis.yml | 38 ++++ .github/workflows/delete_backport_branch.yml | 15 ++ .../draft-release-notes-workflow.yml | 23 +++ .github/workflows/link-checker.yml | 25 +++ .github/workflows/maven-publish.yml | 38 ++++ .github/workflows/sql-pitest.yml | 55 ++++++ .../workflows/sql-test-and-build-workflow.yml | 163 ++++++++++++++++++ .github/workflows/sql-test-workflow.yml | 102 +++++++++++ 10 files changed, 508 insertions(+) create mode 100644 .github/workflows/add-untriaged.yml create mode 100644 .github/workflows/backport.yml create mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/delete_backport_branch.yml create mode 100644 .github/workflows/draft-release-notes-workflow.yml create mode 100644 .github/workflows/link-checker.yml create mode 100644 .github/workflows/maven-publish.yml create mode 100644 .github/workflows/sql-pitest.yml create mode 100644 .github/workflows/sql-test-and-build-workflow.yml create mode 100644 .github/workflows/sql-test-workflow.yml diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml new file mode 100644 index 0000000000..15b9a55651 --- /dev/null +++ b/.github/workflows/add-untriaged.yml @@ -0,0 +1,19 @@ +name: Apply 'untriaged' label during issue lifecycle + +on: + issues: + types: [opened, reopened, transferred] + +jobs: + apply-label: + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v6 + with: + script: | + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ['untriaged'] + }) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml new file mode 100644 index 0000000000..6472a968d8 --- /dev/null +++ b/.github/workflows/backport.yml @@ -0,0 +1,30 @@ +name: Backport +on: + pull_request_target: + types: + - closed + - labeled + +jobs: + backport: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + name: Backport + steps: + - name: GitHub App token + id: github_app_token + uses: tibdex/github-app-token@v1.5.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + installation_id: 22958780 + + - name: Backport + uses: VachaShah/backport@v2.2.0 + with: + github_token: ${{ steps.github_app_token.outputs.token }} + head_template: backport/backport-<%= number %>-to-<%= base %> + failure_labels: backport-failed diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000000..13c4ef0f60 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,38 @@ +name: "CodeQL" + +on: + pull_request: + branches: + - 'main' + - '[1-9]+.[0-9x]+' + push: + branches-ignore: + - 'dependabot/**' + paths: + - '**/*.java' + - '.github/workflows/codeql-analysis.yml' + +jobs: + analyze: + name: CodeQL-Scan + runs-on: ubuntu-latest + permissions: + security-events: write + actions: read + strategy: + fail-fast: false + matrix: + language: [ 'java' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml new file mode 100644 index 0000000000..387a124b8c --- /dev/null +++ b/.github/workflows/delete_backport_branch.yml @@ -0,0 +1,15 @@ +name: Delete merged branch of the backport PRs +on: + pull_request: + types: + - closed + +jobs: + delete-branch: + runs-on: ubuntu-latest + if: startsWith(github.event.pull_request.head.ref,'backport/') + steps: + - name: Delete merged branch + uses: SvanBoxel/delete-merged-branch@main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml new file mode 100644 index 0000000000..858e663d6b --- /dev/null +++ b/.github/workflows/draft-release-notes-workflow.yml @@ -0,0 +1,23 @@ +name: Release Drafter + +on: + pull_request: + push: + branches: + - 'main' + - '[1-9]+.[0-9x]+' + +jobs: + update_release_draft: + name: Update draft release notes + runs-on: ubuntu-latest + steps: + # Drafts your next Release notes as Pull Requests are merged into "develop" + - name: Update draft release notes + uses: release-drafter/release-drafter@v5 + with: + config-name: draft-release-notes-config.yml + tag: (None) + version: 3.0.0.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/link-checker.yml b/.github/workflows/link-checker.yml new file mode 100644 index 0000000000..42c8fd35c8 --- /dev/null +++ b/.github/workflows/link-checker.yml @@ -0,0 +1,25 @@ +name: Link Checker + +on: + pull_request: + push: + branches: + - 'main' + - '[1-9]+.[0-9x]+' + +jobs: + linkchecker: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: lychee Link Checker + id: lychee + uses: lycheeverse/lychee-action@master + with: + args: --accept=200,403,429,999 "./**/*.html" "./**/*.md" "./**/*.txt" --exclude "https://aws.oss.sonatype.*|http://localhost.*|https://localhost|https://odfe-node1:9200/|https://community.tableau.com/docs/DOC-17978|.*family.zzz|opensearch*|.*@amazon.com|.*email.com|.*@github.com|http://timestamp.verisign.com/scripts/timstamp.dll" + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + - name: Fail if there were link errors + run: exit ${{ steps.lychee.outputs.exit_code }} diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml new file mode 100644 index 0000000000..8adf7ae52c --- /dev/null +++ b/.github/workflows/maven-publish.yml @@ -0,0 +1,38 @@ +name: Publish snapshots to maven + +on: + workflow_dispatch: + push: + branches: + - main + - 1.* + - 2.* + +jobs: + build-and-publish-snapshots: + strategy: + fail-fast: false + if: github.repository == 'opensearch-project/sql' + runs-on: ubuntu-latest + + permissions: + id-token: write + contents: write + + steps: + - uses: actions/setup-java@v3 + with: + distribution: temurin # Temurin is a distribution of adoptium + java-version: 11 + - uses: actions/checkout@v3 + - uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }} + aws-region: us-east-1 + - name: publish snapshots to maven + run: | + export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text) + export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text) + echo "::add-mask::$SONATYPE_USERNAME" + echo "::add-mask::$SONATYPE_PASSWORD" + ./gradlew publishPluginZipPublicationToSnapshotsRepository diff --git a/.github/workflows/sql-pitest.yml b/.github/workflows/sql-pitest.yml new file mode 100644 index 0000000000..bc751daefa --- /dev/null +++ b/.github/workflows/sql-pitest.yml @@ -0,0 +1,55 @@ +name: SQL Plugin PiTest + +on: + workflow_dispatch: + inputs: + name: + required: false + type: string + +run-name: + ${{ inputs.name == '' && format('{0} @ {1}', github.ref_name, github.sha) || inputs.name }} + +jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + build-linux: + needs: Get-CI-Image-Tag + strategy: + matrix: + java: + - 11 + - 17 + runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + + steps: + - uses: actions/checkout@v3 + + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + + - name: PiTest with Gradle + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew --continue :core:pitest :opensearch:pitest" + + - name: Upload test reports + if: always() + uses: actions/upload-artifact@v2 + with: + name: test-reports-${{ matrix.entry.java }} + path: | + core/build/reports/** + opensearch/build/reports/** diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml new file mode 100644 index 0000000000..34c45d2108 --- /dev/null +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -0,0 +1,163 @@ +name: SQL Java CI + +on: + pull_request: + push: + branches-ignore: + - 'dependabot/**' + paths: + - '**/*.java' + - '**/*.g4' + - '!sql-jdbc/**' + - '**gradle*' + - '**lombok*' + - 'integ-test/**' + - '**/*.jar' + - '**/*.pom' + - '.github/workflows/sql-test-and-build-workflow.yml' + +jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + build-linux: + needs: Get-CI-Image-Tag + strategy: + # Run all jobs + fail-fast: false + matrix: + java: + - 11 + - 17 + runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + + steps: + - uses: actions/checkout@v3 + + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + + - name: Build with Gradle + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew --continue build" + + - name: Run backward compatibility tests + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./scripts/bwctest.sh" + + - name: Create Artifact Path + run: | + mkdir -p opensearch-sql-builds + cp -r ./plugin/build/distributions/*.zip opensearch-sql-builds/ + + # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action + - name: Upload SQL Coverage Report + if: ${{ always() }} + uses: codecov/codecov-action@v3 + continue-on-error: true + with: + flags: sql-engine + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Upload Artifacts + uses: actions/upload-artifact@v2 + continue-on-error: true + with: + name: opensearch-sql-ubuntu-latest-${{ matrix.java }} + path: opensearch-sql-builds + + - name: Upload test reports + if: ${{ always() }} + uses: actions/upload-artifact@v2 + continue-on-error: true + with: + name: test-reports-ubuntu-latest-${{ matrix.java }} + path: | + sql/build/reports/** + ppl/build/reports/** + core/build/reports/** + common/build/reports/** + opensearch/build/reports/** + integ-test/build/reports/** + protocol/build/reports/** + legacy/build/reports/** + plugin/build/reports/** + doctest/build/testclusters/docTestCluster-0/logs/* + integ-test/build/testclusters/*/logs/* + + build-windows-macos: + strategy: + # Run all jobs + fail-fast: false + matrix: + entry: + - { os: windows-latest, java: 11, os_build_args: -x doctest -PbuildPlatform=windows } + - { os: macos-latest, java: 11} + - { os: windows-latest, java: 17, os_build_args: -x doctest -PbuildPlatform=windows } + - { os: macos-latest, java: 17 } + runs-on: ${{ matrix.entry.os }} + + steps: + - uses: actions/checkout@v3 + + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.entry.java }} + + - name: Build with Gradle + run: ./gradlew --continue build ${{ matrix.entry.os_build_args }} + + - name: Create Artifact Path + run: | + mkdir -p opensearch-sql-builds + cp -r ./plugin/build/distributions/*.zip opensearch-sql-builds/ + + # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action + - name: Upload SQL Coverage Report + if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} + uses: codecov/codecov-action@v3 + continue-on-error: true + with: + flags: sql-engine + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Upload Artifacts + uses: actions/upload-artifact@v2 + continue-on-error: true + with: + name: opensearch-sql-${{ matrix.entry.os }}-${{ matrix.entry.java }} + path: opensearch-sql-builds + + - name: Upload test reports + if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} + uses: actions/upload-artifact@v2 + continue-on-error: true + with: + name: test-reports-${{ matrix.entry.os }}-${{ matrix.entry.java }} + path: | + sql/build/reports/** + ppl/build/reports/** + core/build/reports/** + common/build/reports/** + opensearch/build/reports/** + integ-test/build/reports/** + protocol/build/reports/** + legacy/build/reports/** + plugin/build/reports/** + doctest/build/testclusters/docTestCluster-0/logs/* + integ-test/build/testclusters/*/logs/* diff --git a/.github/workflows/sql-test-workflow.yml b/.github/workflows/sql-test-workflow.yml new file mode 100644 index 0000000000..46c1930cc8 --- /dev/null +++ b/.github/workflows/sql-test-workflow.yml @@ -0,0 +1,102 @@ +name: SQL Plugin Tests + +on: + workflow_dispatch: + inputs: + name: + required: false + type: string + +run-name: + ${{ inputs.name == '' && format('{0} @ {1}', github.ref_name, github.sha) || inputs.name }} + +jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + build: + needs: Get-CI-Image-Tag + strategy: + matrix: + java: + - 11 + - 17 + runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + + steps: + - uses: actions/checkout@v3 + + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + + - name: Run tests + id: tests + run: | + chown -R 1000:1000 `pwd` + # Spotless + su `id -un 1000` -c './gradlew :opensearch:spotlessCheck || echo "* Spotless failed for opensearch" > report.log' + su `id -un 1000` -c './gradlew :sql:spotlessCheck || echo "* Spotless failed for sql" >> report.log' + su `id -un 1000` -c './gradlew :ppl:spotlessCheck || echo "* Spotless failed for ppl" >> report.log' + su `id -un 1000` -c './gradlew :core:spotlessCheck || echo "* Spotless failed for core" >> report.log' + su `id -un 1000` -c './gradlew :common:spotlessCheck || echo "* Spotless failed for common" >> report.log' + su `id -un 1000` -c './gradlew :legacy:spotlessCheck || echo "* Spotless failed for legacy" >> report.log' + su `id -un 1000` -c './gradlew :protocol:spotlessCheck || echo "* Spotless failed for protocol" >> report.log' + su `id -un 1000` -c './gradlew :opensearch-sql-plugin:spotlessCheck || echo "* Spotless failed for plugin" >> report.log' + su `id -un 1000` -c './gradlew :integ-test:spotlessCheck || echo "* Spotless failed for integ-test" >> report.log' + # Unit tests + su `id -un 1000` -c './gradlew :opensearch:test || echo "* Unit tests failed for opensearch" >> report.log' + su `id -un 1000` -c './gradlew :ppl:test || echo "* Unit tests failed for sql" >> report.log' + su `id -un 1000` -c './gradlew :sql:test || echo "* Unit tests failed for ppl" >> report.log' + su `id -un 1000` -c './gradlew :core:test || echo "* Unit tests failed for core" >> report.log' + su `id -un 1000` -c './gradlew :protocol:test || echo "* Unit tests failed for protocol" >> report.log' + su `id -un 1000` -c './gradlew :opensearch-sql-plugin:test || echo "* Unit tests failed for plugin" >> report.log' + su `id -un 1000` -c './gradlew :legacy:test || echo "* Unit tests failed for legacy" >> report.log' + # jacoco + su `id -un 1000` -c './gradlew :opensearch:jacocoTestCoverageVerification || echo "* Jacoco failed for opensearch" >> report.log' + su `id -un 1000` -c './gradlew :ppl:jacocoTestCoverageVerification || echo "* Jacoco failed for sql" >> report.log' + su `id -un 1000` -c './gradlew :sql:jacocoTestCoverageVerification || echo "* Jacoco failed for ppl" >> report.log' + su `id -un 1000` -c './gradlew :core:jacocoTestCoverageVerification || echo "* Jacoco failed for core" >> report.log' + su `id -un 1000` -c './gradlew :protocol:jacocoTestCoverageVerification || echo "* Jacoco failed for protocol" >> report.log' + su `id -un 1000` -c './gradlew :opensearch-sql-plugin:jacocoTestCoverageVerification || echo "* Jacoco failed for plugin" >> report.log' + # Misc tests + su `id -un 1000` -c './gradlew :integ-test:integTest || echo "* Integration test failed" >> report.log' + su `id -un 1000` -c './gradlew :doctest:doctest || echo "* Doctest failed" >> report.log' + su `id -un 1000` -c './scripts/bwctest.sh || echo "* Backward compatibility test failed" >> report.log' + + - name: Verify test results + run: | + if [[ -e report.log ]] + then + echo "## FAILED TESTS :facepalm::warning::bangbang:" >> $GITHUB_STEP_SUMMARY + cat report.log >> $GITHUB_STEP_SUMMARY + exit 1 + fi + + - name: Upload test reports + if: always() + uses: actions/upload-artifact@v2 + with: + name: test-reports-${{ matrix.entry.java }} + path: | + sql/build/reports/** + ppl/build/reports/** + core/build/reports/** + common/build/reports/** + opensearch/build/reports/** + integ-test/build/reports/** + protocol/build/reports/** + legacy/build/reports/** + plugin/build/reports/** + doctest/build/testclusters/docTestCluster-0/logs/* + integ-test/build/testclusters/*/logs/*