diff --git a/.github/workflows/integ-tests-with-geo.yml b/.github/workflows/integ-tests-with-geo.yml new file mode 100644 index 0000000000..065c3b3e3b --- /dev/null +++ b/.github/workflows/integ-tests-with-geo.yml @@ -0,0 +1,89 @@ +name: GeoSpatial Plugin IT + +on: + pull_request: + push: + branches-ignore: + - 'dependabot/**' + paths: + - 'integ-test/**' + - '.github/workflows/integ-tests-with-geo.yml' + +jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + security-it-linux: + needs: Get-CI-Image-Tag + strategy: + fail-fast: false + matrix: + java: [21] + runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - uses: actions/checkout@v4 + + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + + - name: Build with Gradle + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew integTestWithGeo" + + - name: Upload test reports + if: ${{ always() }} + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: test-reports-${{ matrix.os }}-${{ matrix.java }} + path: | + integ-test/build/reports/** + integ-test/build/testclusters/*/logs/* + integ-test/build/testclusters/*/config/* + + security-it-windows-macos: + strategy: + fail-fast: false + matrix: + os: [ windows-latest, macos-13 ] + java: [21] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + + - name: Build with Gradle + run: ./gradlew integTestWithGeo + + - name: Upload test reports + if: ${{ always() }} + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: test-reports-${{ matrix.os }}-${{ matrix.java }} + path: | + integ-test/build/reports/** + integ-test/build/testclusters/*/logs/* + integ-test/build/testclusters/*/config/* diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 798a0be536..3da4f3aae3 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -49,6 +49,12 @@ String bwcVersion = baseVersion + ".0"; String baseName = "sqlBwcCluster" String bwcFilePath = "src/test/resources/bwc/" +Map dummyLoginConfig = [ + https: "false", + user: "admin", + password: "admin" +] + repositories { mavenCentral() maven { url 'https://jitpack.io' } @@ -77,11 +83,24 @@ ext { return repo + "opensearch-security-${securitySnapshotVersion}.zip" } - var projectAbsPath = projectDir.getAbsolutePath() - File downloadedSecurityPlugin = Paths.get(projectAbsPath, 'bin', 'opensearch-security-snapshot.zip').toFile() + configureGeoPlugin = { OpenSearchCluster cluster -> + File downloadedGeoPlugin = Paths.get(projectAbsPath, 'bin', 'opensearch-geospatial-snapshot.zip').toFile() + if (!downloadedGeoPlugin.exists()) { + download.run { + src getPluginDownloadLink("geospatial") + dest downloadedGeoPlugin + } + } else { + println "Geo-spatial Plugin File Already Exists" + } + cluster.plugin provider {(RegularFile) (() -> downloadedGeoPlugin)} + } + configureSecurityPlugin = { OpenSearchCluster cluster -> + File downloadedSecurityPlugin = Paths.get(projectAbsPath, 'bin', 'opensearch-security-snapshot.zip').toFile() + cluster.getNodes().forEach { node -> var creds = node.getCredentials() if (creds.isEmpty()) { @@ -94,7 +113,7 @@ ext { // add a check to avoid re-downloading multiple times during single test run if (!downloadedSecurityPlugin.exists()) { download.run { - src getSecurityPluginDownloadLink() + src getPluginDownloadLink("opensearch-security") dest downloadedSecurityPlugin } } else { @@ -134,8 +153,7 @@ ext { ].forEach { name, value -> cluster.setting name, value } - - cluster.plugin provider((Callable) (() -> (RegularFile) (() -> downloadedSecurityPlugin))) + cluster.plugin provider {(RegularFile) (() -> downloadedSecurityPlugin)} } bwcOpenSearchJSDownload = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + baseVersion + '/latest/linux/x64/tar/builds/' + @@ -224,21 +242,39 @@ testClusters.all { } def getJobSchedulerPlugin() { - provider(new Callable() { - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return configurations.zipArchive.asFileTree.matching { - include '**/opensearch-job-scheduler*' - }.singleFile - } - } - } - }) + provider { (RegularFile) (() -> + configurations.zipArchive.asFileTree.matching { + include '**/opensearch-job-scheduler*' + }.singleFile ) + } +} + +static def getAllHttpSocketURI(cluster) { + return cluster.nodes.stream() + .flatMap { node -> node.getAllHttpSocketURI().stream() } + .collect(Collectors.joining(",")) } +static def getAllTransportSocketURI(cluster) { + return cluster.nodes.stream() + .flatMap { node -> node.getAllTransportPortURI().stream() } + .collect(Collectors.joining(",")) +} + +def getPluginDownloadLink(pluginName) { + var repo = "https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/plugin/" + + pluginName + "/$opensearch_build_snapshot/" + var metadataFile = Paths.get(projectDir.toString(), "build", "maven-metadata.xml").toAbsolutePath().toFile() + download.run { + src repo + "maven-metadata.xml" + dest metadataFile + } + def metadata = new XmlParser().parse(metadataFile) + def PluginSnapshotVersion = metadata.versioning.snapshotVersions[0].snapshotVersion[0].value[0].text() + return repo + pluginName +"-${PluginSnapshotVersion}.zip" +} + + testClusters { integTest { testDistribution = 'archive' @@ -261,6 +297,11 @@ testClusters { plugin(getJobSchedulerPlugin()) plugin ":opensearch-sql-plugin" } + integTestWithGeo { + testDistribution = 'archive' + plugin(getJobSchedulerPlugin()) + plugin ":opensearch-sql-plugin" + } } task startPrometheus(type: SpawnProcessTask) { @@ -370,21 +411,51 @@ task integTestWithSecurity(type: RestIntegTestTask) { doFirst { systemProperty 'cluster.debug', getDebug() getClusters().forEach { cluster -> + systemProperty "tests.rest.${cluster.name}.http_hosts", "${-> getAllHttpSocketURI(cluster)}" + systemProperty "tests.rest.${cluster.name}.transport_hosts", "${-> getAllTransportSocketURI(cluster)}" + } + systemProperties dummyLoginConfig + } - String allTransportSocketURI = cluster.nodes.stream().flatMap { node -> - node.getAllTransportPortURI().stream() - }.collect(Collectors.joining(",")) - String allHttpSocketURI = cluster.nodes.stream().flatMap { node -> - node.getAllHttpSocketURI().stream() - }.collect(Collectors.joining(",")) + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' + } + + // NOTE: this IT config discovers only junit5 (jupiter) tests. + // https://github.com/opensearch-project/sql/issues/1974 + filter { + includeTestsMatching 'org.opensearch.sql.security.CrossClusterSearchIT' + } +} - systemProperty "tests.rest.${cluster.name}.http_hosts", "${-> allHttpSocketURI}" - systemProperty "tests.rest.${cluster.name}.transport_hosts", "${-> allTransportSocketURI}" - } - systemProperty "https", "false" - systemProperty "user", "admin" - systemProperty "password", "admin" +task integTestWithGeo(type: RestIntegTestTask) { + useCluster testClusters.remoteCluster + + getClusters().forEach { cluster -> + configureGeoPlugin(cluster) + } + + useJUnitPlatform() + dependsOn ':opensearch-sql-plugin:bundlePlugin' + testLogging { + events "passed", "skipped", "failed" + } + + systemProperty 'tests.security.manager', 'false' + systemProperty 'project.root', project.projectDir.absolutePath + // Set default query size limit + systemProperty 'defaultQuerySizeLimit', '10000' + + // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for + // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time. + doFirst { + systemProperty 'cluster.debug', getDebug() + getClusters().forEach { cluster -> + systemProperty "tests.rest.${cluster.name}.http_hosts", "${-> getAllHttpSocketURI(cluster)}" + systemProperty "tests.rest.${cluster.name}.transport_hosts", "${-> getAllTransportSocketURI(cluster)}" + } + systemProperties dummyLoginConfig } if (System.getProperty("test.debug") != null) { @@ -394,10 +465,11 @@ task integTestWithSecurity(type: RestIntegTestTask) { // NOTE: this IT config discovers only junit5 (jupiter) tests. // https://github.com/opensearch-project/sql/issues/1974 filter { - includeTestsMatching 'org.opensearch.sql.security.CrossClusterSearchIT' + includeTestsMatching 'org.opensearch.sql.geo.PplIpEnrichmentIT' } } + // Run PPL ITs and new, legacy and comparison SQL ITs with new SQL engine enabled integTest { useCluster testClusters.remoteCluster @@ -405,15 +477,8 @@ integTest { // Set properties for connection to clusters and between clusters doFirst { getClusters().forEach { cluster -> - String allTransportSocketURI = cluster.nodes.stream().flatMap { node -> - node.getAllTransportPortURI().stream() - }.collect(Collectors.joining(",")) - String allHttpSocketURI = cluster.nodes.stream().flatMap { node -> - node.getAllHttpSocketURI().stream() - }.collect(Collectors.joining(",")) - - systemProperty "tests.rest.${cluster.name}.http_hosts", "${-> allHttpSocketURI}" - systemProperty "tests.rest.${cluster.name}.transport_hosts", "${-> allTransportSocketURI}" + systemProperty "tests.rest.${cluster.name}.http_hosts", "${-> getAllHttpSocketURI(cluster)}" + systemProperty "tests.rest.${cluster.name}.transport_hosts", "${-> getAllTransportSocketURI(cluster)}" } } @@ -480,8 +545,9 @@ integTest { // Exclude JDBC related tests exclude 'org/opensearch/sql/jdbc/**' - // Exclude this IT, because they executed in another task (:integTestWithSecurity) + // Exclude these IT, because they executed in respective plugin-related tasks (ex: :integTestWithSecurity) exclude 'org/opensearch/sql/security/**' + exclude 'org/opensearch/sql/geo/**' } @@ -526,37 +592,22 @@ task comparisonTest(type: RestIntegTestTask) { testDistribution = "ARCHIVE" versions = [baseVersion, opensearch_version] numberOfNodes = 3 - plugin(provider(new Callable(){ - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - if (new File("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion").exists()) { - project.delete(files("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion")) - } - project.mkdir bwcJobSchedulerPath + bwcVersion - ant.get(src: bwcOpenSearchJSDownload, - dest: bwcJobSchedulerPath + bwcVersion, - httpusecaches: false) - return fileTree(bwcJobSchedulerPath + bwcVersion).getSingleFile() - } + plugin(provider { (RegularFile) ({ + if (new File("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion").exists()) { + project.delete(files("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion")) } - } - })) - plugin(provider(new Callable(){ - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return configurations.zipArchive.asFileTree.matching { - include '**/opensearch-sql-plugin*' - }.singleFile - } - } - } - })) + project.mkdir bwcJobSchedulerPath + bwcVersion + ant.get(src: bwcOpenSearchJSDownload, + dest: bwcJobSchedulerPath + bwcVersion, + httpusecaches: false) + fileTree(bwcJobSchedulerPath + bwcVersion).getSingleFile() + }) + }) + plugin(provider { (RegularFile) ( () -> + configurations.zipArchive.asFileTree.matching { + include '**/opensearch-sql-plugin*' + }.singleFile ) + }) setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' } @@ -565,17 +616,9 @@ task comparisonTest(type: RestIntegTestTask) { List> plugins = [ getJobSchedulerPlugin(), - provider(new Callable() { - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return fileTree(bwcFilePath + project.version).getSingleFile() - } - } - } - }) + provider{ (RegularFile) ( + fileTree(bwcFilePath + project.version).getSingleFile()) + } ] // Creates 2 test clusters with 3 nodes of the old version. diff --git a/integ-test/src/test/java/org/opensearch/sql/geo/PplIpEnrichmentIT.java b/integ-test/src/test/java/org/opensearch/sql/geo/PplIpEnrichmentIT.java new file mode 100644 index 0000000000..1f07813170 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/geo/PplIpEnrichmentIT.java @@ -0,0 +1,49 @@ +/* + * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.geo; + +import static org.opensearch.sql.legacy.TestUtils.getResponseBody; + +import java.io.IOException; +import lombok.SneakyThrows; +import org.junit.Assert; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** IP enrichment PPL request with OpenSearch Geo-sptial plugin */ +public class PplIpEnrichmentIT extends PPLIntegTestCase { + + private static boolean initialized = false; + + private static String PLUGIN_NAME = "opensearch-geospatial"; + + @SneakyThrows + @BeforeEach + public void initialize() { + if (!initialized) { + setUpIndices(); + initialized = true; + } + } + + @Test + public void testGeoPluginInstallation() throws IOException { + + Request request = new Request("GET", "/_cat/plugins?v"); + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + Response response = client().performRequest(request); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + Assert.assertTrue(getResponseBody(response, true).contains(PLUGIN_NAME)); + } +}