From 08151df88d9f15a079c8e5c4d3a0ba666b7a4c5b Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 30 Oct 2023 14:29:29 -0400 Subject: [PATCH] Onboarding jenkins prod docker images to github actions Signed-off-by: Peter Zhu --- .github/workflows/benchmark.yml | 20 +++++-- .../workflows/test_build_multi_platform.yml | 54 +++++++++++++++++-- .github/workflows/test_bwc.yml | 22 ++++++-- 3 files changed, 86 insertions(+), 10 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c5caeedd0..edf7a0bbb 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -8,7 +8,13 @@ on: - "*" jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + Build-ad: + needs: Get-CI-Image-Tag strategy: matrix: java: [17] @@ -16,18 +22,26 @@ jobs: name: Run Anomaly detection model performance benchmark runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root steps: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: 'temurin' java-version: ${{ matrix.java }} # anomaly-detection - name: Checkout AD - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Build and Run Tests run: | - ./gradlew ':test' --tests "org.opensearch.ad.ml.HCADModelPerfTests" -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true - ./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true \ No newline at end of file + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew ':test' --tests 'org.opensearch.ad.ml.HCADModelPerfTests' -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true" + su `id -un 1000` -c "./gradlew integTest --tests 'org.opensearch.ad.e2e.SingleStreamModelPerfIT' -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true" diff --git a/.github/workflows/test_build_multi_platform.yml b/.github/workflows/test_build_multi_platform.yml index db0e1dfc0..ac81a3697 100644 --- a/.github/workflows/test_build_multi_platform.yml +++ b/.github/workflows/test_build_multi_platform.yml @@ -8,6 +8,11 @@ on: - "*" jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + Build-ad-windows: strategy: matrix: @@ -20,6 +25,7 @@ jobs: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: 'temurin' java-version: ${{ matrix.java }} - name: Checkout Anomaly Detection uses: actions/checkout@v4 @@ -38,15 +44,56 @@ jobs: with: file: ./build/reports/jacoco/test/jacocoTestReport.xml flags: plugin - Build-ad: + + Build-ad-linux: + needs: Get-CI-Image-Tag + strategy: + matrix: + java: [11, 17, 20] + fail-fast: false + name: Build and Test Anomaly detection Plugin + runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + + env: + JENKINS_URL: build.ci.opensearch.org + + steps: + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + + - name: Checkout AD + uses: actions/checkout@v3 + + - name: Assemble / build / mavenlocal / integTest + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew assemble && + ./gradlew build && + ./gradlew publishToMavenLocal && + ./gradlew integTest -PnumNodes=3" + - name: Upload Coverage Report + uses: codecov/codecov-action@v3 + with: + file: ./build/reports/jacoco/test/jacocoTestReport.xml + flags: plugin + + Build-ad-macos: strategy: matrix: java: [11,17,20] - os: [ubuntu-latest, macos-latest] fail-fast: false name: Build and Test Anomaly detection Plugin - runs-on: ${{ matrix.os }} + runs-on: macos-latest env: JENKINS_URL: build.ci.opensearch.org @@ -54,6 +101,7 @@ jobs: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: 'temurin' java-version: ${{ matrix.java }} - name: Checkout AD diff --git a/.github/workflows/test_bwc.yml b/.github/workflows/test_bwc.yml index 3c86f238a..8c9d68133 100644 --- a/.github/workflows/test_bwc.yml +++ b/.github/workflows/test_bwc.yml @@ -8,7 +8,12 @@ on: - "*" jobs: - Build-ad: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + Build-ad-linux: strategy: matrix: java: [11,17,20] @@ -16,22 +21,30 @@ jobs: name: Test Anomaly detection BWC runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root steps: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: 'temurin' java-version: ${{ matrix.java }} # anomaly-detection - name: Checkout AD - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Assemble anomaly-detection run: | plugin_version=`./gradlew properties -q | grep "opensearch_build:" | awk '{print $2}'` echo plugin_version $plugin_version - ./gradlew assemble + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew assemble" echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..." mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..." @@ -42,4 +55,5 @@ jobs: - name: Run AD Backwards Compatibility Tests run: | echo "Running backwards compatibility tests ..." - ./gradlew bwcTestSuite -Dtests.security.manager=false + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew bwcTestSuite -Dtests.security.manager=false"