[Backport 2.x] Excluding delete system index tests from a security enabled cluster

.github/workflows/test_security.yml

@@ -0,0 +1,43 @@
+name: Run Security tests
+on:
+  push:
+    branches-ignore:
+      - 'whitesource-remediate/**'
+      - 'backport/**'
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  integ-test-with-security-linux:
+    strategy:
+      matrix:
+        java: [11, 17, 21]
+
+    name: Run Security Integration Tests on Linux
+    runs-on: ubuntu-latest
+    needs: Get-CI-Image-Tag
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
+    steps:
+      - name: Checkout Skills
+        uses: actions/checkout@v3
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java }}
+      - name: Run tests
+        # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip.
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true"

build.gradle

@@ -5,6 +5,8 @@
 
 import org.opensearch.gradle.test.RestIntegTestTask
 import java.util.concurrent.Callable
+import org.opensearch.gradle.testclusters.OpenSearchCluster
+import java.nio.file.Paths
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
 
 buildscript {
@@ -43,6 +45,7 @@ plugins {
     id 'java-library'
     id 'com.diffplug.spotless' version '6.23.0'
     id "io.freefair.lombok" version "8.4"
+    id "de.undercouch.download" version "5.3.0"
 }
 
 lombok {
@@ -81,6 +84,7 @@ def adJarDirectory = "$buildDir/dependencies/opensearch-anomaly-detection"
 
 configurations {
     zipArchive
+    secureIntegTestPluginArchive
     all {
         resolutionStrategy {
             force "org.mockito:mockito-core:${versions.mockito}"
@@ -136,6 +140,7 @@ dependencies {
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-knn', version: "${opensearch_build}"
     zipArchive group: 'org.opensearch.plugin', name:'neural-search', version: "${opensearch_build}"
     zipArchive group: 'org.opensearch.plugin', name:'alerting', version: "${opensearch_build}"
+    secureIntegTestPluginArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}"
 
     // Test dependencies
     testImplementation "org.opensearch.test:framework:${opensearch_version}"
@@ -240,6 +245,14 @@ opensearchplugin {
     noticeFile rootProject.file("NOTICE")
 }
 
+def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
+opensearch_tmp_dir.mkdirs()
+
+ext {
+    projectSubstitutions = [:]
+    isSnapshot = "true" == System.getProperty("build.snapshot", "true")
+}
+
 allprojects {
     // Default to the apache license
     project.ext.licenseName = 'The Apache Software License, Version 2.0'
@@ -313,8 +326,7 @@ publishing {
     gradle.startParameter.setLogLevel(LogLevel.DEBUG)
 }
 
-def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
-opensearch_tmp_dir.mkdirs()
+
 def _numNodes = findProperty('numNodes') as Integer ?: 1
 
 // Set up integration tests
@@ -335,6 +347,35 @@ integTest {
     systemProperty "user", System.getProperty("user")
     systemProperty "password", System.getProperty("password")
 
+    systemProperty 'security.enabled', System.getProperty('security.enabled')
+    var is_https = System.getProperty("https")
+    var user = System.getProperty("user")
+    var password = System.getProperty("password")
+
+    if (System.getProperty("security.enabled") != null) {
+        // If security is enabled, set is_https/user/password defaults
+        // admin password is permissable here since the security plugin is manually configured using the default internal_users.yml configuration
+        is_https = is_https == null ? "true" : is_https
+        user = user == null ? "admin" : user
+        password = password == null ? "admin" : password
+        System.setProperty("https", is_https)
+        System.setProperty("user", user)
+        System.setProperty("password", password)
+    }
+
+    systemProperty("https", is_https)
+    systemProperty("user", user)
+    systemProperty("password", password)
+
+    if (System.getProperty("https") != null && System.getProperty("https") == "true") {
+        filter {
+            excludeTestsMatching "org.opensearch.integTest.SearchAlertsToolIT"
+            excludeTestsMatching "org.opensearch.integTest.SearchAnomalyDetectorsToolIT"
+            excludeTestsMatching "org.opensearch.integTest.SearchAnomalyResultsToolIT"
+            excludeTestsMatching "org.opensearch.integTest.SearchMonitorsToolIT"
+        }
+    }
+
 
     // doFirst delays this block until execution time
     doFirst {
@@ -361,6 +402,96 @@ integTest {
 testClusters.integTest {
     testDistribution = "ARCHIVE"
 
+    // Optionally install security
+    if (System.getProperty("security.enabled") != null) {
+         configurations.secureIntegTestPluginArchive.asFileTree.each {
+            if(it.name.contains("opensearch-security")){
+                plugin(provider(new Callable<RegularFile>() {
+                    @Override
+                    RegularFile call() throws Exception {
+                        return new RegularFile() {
+                            @Override
+                            File getAsFile() {
+                                return it
+                            }
+                        }
+                    }
+                }))
+            }
+        }
+
+        getNodes().forEach { node ->
+            var creds = node.getCredentials()
+            // admin password is permissable here since the security plugin is manually configured using the default internal_users.yml configuration
+            if (creds.isEmpty()) {
+                creds.add(Map.of('username', 'admin', 'password', 'admin'))
+            } else {
+                creds.get(0).putAll(Map.of('username', 'admin', 'password', 'admin'))
+            }
+        }
+
+        // Config below including files are copied from security demo configuration
+        ['esnode.pem', 'esnode-key.pem', 'root-ca.pem','kirk.pem','kirk-key.pem'].forEach { file ->
+            File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile()
+            download.run {
+                src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file
+                dest local
+                overwrite false
+            }
+        }
+
+        // // Config below including files are copied from security demo configuration
+        extraConfigFile("esnode.pem", file("$opensearch_tmp_dir/esnode.pem"))
+        extraConfigFile("esnode-key.pem", file("$opensearch_tmp_dir/esnode-key.pem"))
+        extraConfigFile("root-ca.pem", file("$opensearch_tmp_dir/root-ca.pem"))
+
+        // This configuration is copied from the security plugins demo install:
+        // https://github.com/opensearch-project/security/blob/2.11.1.0/tools/install_demo_configuration.sh#L365-L388
+        setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
+        setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
+        setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
+        setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
+        setting("plugins.security.ssl.http.enabled", "true")
+        setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
+        setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
+        setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
+        setting("plugins.security.allow_unsafe_democertificates", "true")
+        setting("plugins.security.allow_default_init_securityindex", "true")
+        setting("plugins.security.unsupported.inject_user.enabled", "true")
+
+        setting("plugins.security.authcz.admin_dn", "\n- CN=kirk,OU=client,O=client,L=test, C=de")
+        setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]')
+        setting('plugins.security.system_indices.enabled', "true")
+        setting('plugins.security.system_indices.indices', '[' +
+                '".plugins-ml-config", ' +
+                '".plugins-ml-connector", ' +
+                '".plugins-ml-model-group", ' +
+                '".plugins-ml-model", ".plugins-ml-task", ' +
+                '".plugins-ml-conversation-meta", ' +
+                '".plugins-ml-conversation-interactions", ' +
+                '".opendistro-alerting-config", ' +
+                '".opendistro-alerting-alert*", ' +
+                '".opendistro-anomaly-results*", ' +
+                '".opendistro-anomaly-detector*", ' +
+                '".opendistro-anomaly-checkpoints", ' +
+                '".opendistro-anomaly-detection-state", ' +
+                '".opendistro-reports-*", ' +
+                '".opensearch-notifications-*", ' +
+                '".opensearch-notebooks", ' +
+                '".opensearch-observability", ' +
+                '".ql-datasources", ' +
+                '".opendistro-asynchronous-search-response*", ' +
+                '".replication-metadata-store", ' +
+                '".opensearch-knn-models", ' +
+                '".geospatial-ip2geo-data*", ' +
+                '".plugins-flow-framework-config", ' +
+                '".plugins-flow-framework-templates", ' +
+                '".plugins-flow-framework-state"' +
+                ']'
+        )
+        setSecure(true)
+    }
+
     // Installs all registered zipArchive dependencies on integTest cluster nodes
     configurations.zipArchive.asFileTree.each {
         plugin(provider(new Callable<RegularFile>(){
@@ -412,6 +543,15 @@ task integTestRemote(type: RestIntegTestTask) {
             includeTestsMatching "org.opensearch.integTest.*IT"
         }
     }
+
+    if (System.getProperty("https") != null && System.getProperty("https") == "true") {
+        filter {
+            excludeTestsMatching "org.opensearch.integTest.SearchAlertsToolIT"
+            excludeTestsMatching "org.opensearch.integTest.SearchAnomalyDetectorsToolIT"
+            excludeTestsMatching "org.opensearch.integTest.SearchAnomalyResultsToolIT"
+            excludeTestsMatching "org.opensearch.integTest.SearchMonitorsToolIT"
+        }
+    }
 }
 
 // Automatically sets up the integration test cluster locally