[7.x] Convert second 1/2 x-pack plugins from integTest to [yaml | java]RestTest or internalClusterTest (#61802) (#61856)

For 1/2 the plugins in x-pack, the integTest
task is now a no-op and all of the tests are now executed via a test,
yamlRestTest, javaRestTest, or internalClusterTest.

This includes the following projects:
security, spatial, stack, transform, vecotrs, voting-only-node, and watcher.

A few of the more specialized qa projects within these plugins
have not been changed with this PR due to additional complexity which should
be addressed separately. 

related: #60630
related: #56841
related: #59939
related: #55896

x-pack/plugin/searchable-snapshots/build.gradle

@@ -1,4 +1,6 @@
 import org.elasticsearch.gradle.info.BuildParams
+
+apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.esplugin'
 esplugin {
   name 'searchable-snapshots'
@@ -10,7 +12,7 @@ archivesBaseName = 'x-pack-searchable-snapshots'
 
 dependencies {
   compileOnly project(path: xpackModule('core'), configuration: 'default')
-  testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
+  internalClusterTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
 // xpack modules are installed in real clusters as the meta plugin, so

x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java

@@ -47,6 +47,7 @@ import org.elasticsearch.xpack.searchablesnapshots.action.SearchableSnapshotsSta
 import org.elasticsearch.xpack.searchablesnapshots.action.SearchableSnapshotsStatsRequest;
 import org.elasticsearch.xpack.searchablesnapshots.action.SearchableSnapshotsStatsResponse;
 import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService;
+import org.hamcrest.Matchers;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -78,7 +79,6 @@ import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegTestCase {
@@ -737,7 +737,7 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT
             new SearchableSnapshotsStatsRequest(indexName)
         ).actionGet();
         final NumShards restoredNumShards = getNumShards(indexName);
-        assertThat(statsResponse.getStats(), hasSize(restoredNumShards.totalNumShards));
+        assertThat(statsResponse.getStats(), Matchers.hasSize(restoredNumShards.totalNumShards));
 
         final long totalSize = statsResponse.getStats()
             .stream()

x-pack/plugin/security/build.gradle

@@ -2,6 +2,7 @@ import org.elasticsearch.gradle.info.BuildParams
 
 apply plugin: 'elasticsearch.esplugin'
 apply plugin: 'elasticsearch.publish'
+apply plugin: 'elasticsearch.internal-cluster-test'
 esplugin {
   name 'x-pack-security'
   description 'Elasticsearch Expanded Pack Plugin - Security'
@@ -21,6 +22,7 @@ dependencies {
   testImplementation project(path: xpackModule('sql:sql-action'))
 
   testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
+  internalClusterTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
 
   api 'com.unboundid:unboundid-ldapsdk:4.0.8'
 
@@ -134,11 +136,17 @@ dependencies {
 compileJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked"
 compileTestJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked"
 
+processInternalClusterTestResources {
+  from(project(xpackModule('core')).file('src/main/config'))
+  from(project(xpackModule('core')).file('src/test/resources'))
+}
+
 processTestResources {
   from(project(xpackModule('core')).file('src/main/config'))
   from(project(xpackModule('core')).file('src/test/resources'))
 }
 
+
 configurations {
   testArtifacts.extendsFrom testRuntime
   testArtifacts.extendsFrom testImplementation
@@ -464,7 +472,7 @@ if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
   )
 }
 
-test {
+internalClusterTest {
   /*
    * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
    * other if we allow them to set the number of available processors as it's set-once in Netty.

x-pack/plugin/security/qa/basic-enable-security/build.gradle

@@ -1,21 +1,20 @@
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
+import org.elasticsearch.gradle.test.rest.JavaRestTestPlugin
 
-apply plugin: 'elasticsearch.testclusters'
-apply plugin: 'elasticsearch.standalone-rest-test'
-apply plugin: 'elasticsearch.rest-test'
+apply plugin: 'elasticsearch.java-rest-test'
 
 dependencies {
-  testImplementation project(path: xpackModule('core'), configuration: 'default')
-  testImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
-  testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
+  javaRestTestImplementation project(path: xpackModule('core'), configuration: 'default')
+  javaRestTestImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
+  javaRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
-integTest {
+javaRestTest {
   description = "Run tests against a cluster that doesn't have security"
   systemProperty 'tests.has_security', 'false'
 }
 
-testClusters.integTest {
+testClusters.javaRestTest {
   testDistribution = 'DEFAULT'
   numberOfNodes = 2
   setting 'xpack.ml.enabled', 'false'
@@ -23,14 +22,15 @@ testClusters.integTest {
   setting 'xpack.security.enabled', 'false'
 }
 
-task integTestSecurity(type: StandaloneRestIntegTestTask) {
-  description = "Run tests against a cluster that has security"
-  useCluster testClusters.integTest
-  dependsOn integTest
+task javaRestTestWithSecurity(type: StandaloneRestIntegTestTask) {
+  description = "Run tests against a cluster that has security enabled"
+  useCluster testClusters.javaRestTest
+  dependsOn javaRestTest
   systemProperty 'tests.has_security', 'true'
-
+  testClassesDirs = sourceSets.javaRestTest.output.classesDirs
+  classpath = sourceSets.javaRestTest.runtimeClasspath
   doFirst {
-    testClusters.integTest {
+    testClusters.javaRestTest {
       // Reconfigure cluster to enable security
       setting 'xpack.security.enabled', 'true'
       setting 'xpack.security.authc.anonymous.roles', 'anonymous'
@@ -40,18 +40,18 @@ task integTestSecurity(type: StandaloneRestIntegTestTask) {
       setting 'xpack.security.transport.ssl.key_passphrase', 'transport-password'
       setting 'xpack.security.transport.ssl.certificate_authorities', 'ca.crt'
 
-      extraConfigFile 'transport.key', file('src/test/resources/ssl/transport.key')
-      extraConfigFile 'transport.crt', file('src/test/resources/ssl/transport.crt')
-      extraConfigFile 'ca.crt', file('src/test/resources/ssl/ca.crt')
-      extraConfigFile 'roles.yml', file('src/test/resources/roles.yml')
+      extraConfigFile 'transport.key', file('src/javaRestTest/resources/ssl/transport.key')
+      extraConfigFile 'transport.crt', file('src/javaRestTest/resources/ssl/transport.crt')
+      extraConfigFile 'ca.crt', file('src/javaRestTest/resources/ssl/ca.crt')
+      extraConfigFile 'roles.yml', file('src/javaRestTest/resources/roles.yml')
 
       user username: "admin_user", password: "admin-password"
       user username: "security_test_user", password: "security-test-password", role: "security_test_role"
 
       restart()
     }
-    nonInputProperties.systemProperty 'tests.rest.cluster', "${-> testClusters.integTest.getAllHttpSocketURI().join(",")}"
+    nonInputProperties.systemProperty 'tests.rest.cluster', "${-> testClusters.javaRestTest.getAllHttpSocketURI().join(",")}"
   }
 }
-tasks.named("check").configure { dependsOn(integTestSecurity) }
+tasks.named("check").configure { dependsOn(javaRestTestWithSecurity) }
 

x-pack/plugin/security/qa/security-basic/build.gradle

@@ -1,14 +1,13 @@
-apply plugin: 'elasticsearch.testclusters'
-apply plugin: 'elasticsearch.standalone-rest-test'
-apply plugin: 'elasticsearch.rest-test'
+
+apply plugin: 'elasticsearch.java-rest-test'
 
 dependencies {
-  testImplementation project(path: xpackModule('core'), configuration: 'default')
-  testImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
-  testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
+  javaRestTestImplementation project(path: xpackModule('core'), configuration: 'default')
+  javaRestTestImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
+  javaRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
-testClusters.integTest {
+testClusters.all {
   testDistribution = 'DEFAULT'
   numberOfNodes = 2
 
@@ -21,7 +20,7 @@ testClusters.integTest {
   setting 'xpack.security.authc.token.enabled', 'true'
   setting 'xpack.security.authc.api_key.enabled', 'true'
 
-  extraConfigFile 'roles.yml', file('src/test/resources/roles.yml')
+  extraConfigFile 'roles.yml', file('src/javaRestTest/resources/roles.yml')
   user username: "admin_user", password: "admin-password"
   user username: "security_test_user", password: "security-test-password", role: "security_test_role"
 }

x-pack/plugin/security/qa/security-disabled/build.gradle

@@ -6,17 +6,15 @@
  *              defined, it should be not fail
  */
 
-apply plugin: 'elasticsearch.testclusters'
-apply plugin: 'elasticsearch.standalone-rest-test'
-apply plugin: 'elasticsearch.rest-test'
+apply plugin: 'elasticsearch.java-rest-test'
 
 dependencies {
-  testImplementation project(path: xpackModule('core'), configuration: 'default')
-  testImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
-  testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
+  javaRestTestImplementation project(path: xpackModule('core'), configuration: 'default')
+  javaRestTestImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
+  javaRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
-testClusters.integTest {
+testClusters.all {
   testDistribution = 'DEFAULT'
   numberOfNodes = 2
 

x-pack/plugin/security/qa/security-not-enabled/build.gradle

@@ -6,17 +6,15 @@
  *              defined, it should be not fail
  */
 
-apply plugin: 'elasticsearch.testclusters'
-apply plugin: 'elasticsearch.standalone-rest-test'
-apply plugin: 'elasticsearch.rest-test'
+apply plugin: 'elasticsearch.java-rest-test'
 
 dependencies {
-  testImplementation project(path: xpackModule('core'), configuration: 'default')
-  testImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
-  testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
+  javaRestTestImplementation project(path: xpackModule('core'), configuration: 'default')
+  javaRestTestImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
+  javaRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
-testClusters.integTest {
+testClusters.all {
   testDistribution = 'DEFAULT'
   numberOfNodes = 2
 

x-pack/plugin/security/qa/security-trial/build.gradle

@@ -1,14 +1,12 @@
-apply plugin: 'elasticsearch.testclusters'
-apply plugin: 'elasticsearch.standalone-rest-test'
-apply plugin: 'elasticsearch.rest-test'
+apply plugin: 'elasticsearch.java-rest-test'
 
 dependencies {
-  testImplementation project(path: xpackModule('core'), configuration: 'default')
-  testImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
-  testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
+  javaRestTestImplementation project(path: xpackModule('core'), configuration: 'default')
+  javaRestTestImplementation project(path: xpackModule('security'), configuration: 'testArtifacts')
+  javaRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
-testClusters.integTest {
+testClusters.javaRestTest {
   testDistribution = 'DEFAULT'
   numberOfNodes = 2
 
@@ -21,7 +19,7 @@ testClusters.integTest {
   setting 'xpack.security.authc.token.enabled', 'true'
   setting 'xpack.security.authc.api_key.enabled', 'true'
 
-  extraConfigFile 'roles.yml', file('src/test/resources/roles.yml')
+  extraConfigFile 'roles.yml', file('src/javaRestTest/resources/roles.yml')
   user username: "admin_user", password: "admin-password"
   user username: "security_test_user", password: "security-test-password", role: "security_test_role"
 }

x-pack/plugin/security/qa/tls-basic/build.gradle

@@ -1,6 +1,4 @@
-apply plugin: 'elasticsearch.testclusters'
-apply plugin: 'elasticsearch.standalone-rest-test'
-apply plugin: 'elasticsearch.rest-test'
+apply plugin: 'elasticsearch.java-rest-test'
 
 dependencies {
   testImplementation project(path: xpackModule('core'), configuration: 'default')
@@ -8,21 +6,15 @@ dependencies {
   testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
-forbiddenPatterns {
-  exclude '**/*.key'
-  exclude '**/*.p12'
-}
-
-
-testClusters.integTest {
+testClusters.javaRestTest {
   testDistribution = 'DEFAULT'
   numberOfNodes = 2
 
-  extraConfigFile 'http.key', file('src/test/resources/ssl/http.key')
-  extraConfigFile 'http.crt', file('src/test/resources/ssl/http.crt')
-  extraConfigFile 'transport.key', file('src/test/resources/ssl/transport.key')
-  extraConfigFile 'transport.crt', file('src/test/resources/ssl/transport.crt')
-  extraConfigFile 'ca.crt', file('src/test/resources/ssl/ca.crt')
+  extraConfigFile 'http.key', file('src/javaRestTest/resources/ssl/http.key')
+  extraConfigFile 'http.crt', file('src/javaRestTest/resources/ssl/http.crt')
+  extraConfigFile 'transport.key', file('src/javaRestTest/resources/ssl/transport.key')
+  extraConfigFile 'transport.crt', file('src/javaRestTest/resources/ssl/transport.crt')
+  extraConfigFile 'ca.crt', file('src/javaRestTest/resources/ssl/ca.crt')
 
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'basic'

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java

@@ -162,6 +162,7 @@ public class KibanaUserRoleIntegTests extends NativeRealmIntegTestCase {
         MappingMetadata mappingMetadata = mappingsMap.get(index).get(type);
         assertThat(mappingMetadata.getSourceAsMap(), hasKey("properties"));
         assertThat(mappingMetadata.getSourceAsMap().get("properties"), instanceOf(Map.class));
+        @SuppressWarnings("unchecked")
         Map<String, Object> propertiesMap = (Map<String, Object>) mappingMetadata.getSourceAsMap().get("properties");
         assertThat(propertiesMap, hasKey(field));
     }

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/transport/SecurityServerTransportServiceTests.java

@@ -22,6 +22,7 @@ public class SecurityServerTransportServiceTests extends SecurityIntegTestCase {
 
     public void testSecurityServerTransportServiceWrapsAllHandlers() {
         for (TransportService transportService : internalCluster().getInstances(TransportService.class)) {
+            @SuppressWarnings("rawtypes")
             RequestHandlerRegistry handler = transportService.transport.getRequestHandlers()
                 .getHandler(TransportService.HANDSHAKE_ACTION_NAME);
             assertEquals(

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java

@@ -571,6 +571,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         Client client = client().filterWithHeader(headers);
         SecurityClient securityClient = new SecurityClient(client);
         PlainActionFuture<GetApiKeyResponse> listener = new PlainActionFuture<>();
+        @SuppressWarnings("unchecked")
         List<CreateApiKeyResponse> responses = randomFrom(createApiKeyResponses1, createApiKeyResponses2);
         securityClient.getApiKey(GetApiKeyRequest.usingApiKeyName(responses.get(0).getName(), false), listener);
         verifyGetResponse(1, responses, listener.get(), Collections.singleton(responses.get(0).getId()), null);
@@ -651,6 +652,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         final List<CreateApiKeyResponse> userWithManageOwnApiKeyRoleApiKeys = createApiKeys("user_with_manage_own_api_key_role",
             "user_with_run_as_role", noOfApiKeysForUserWithManageApiKeyRole, null, "monitor");
         PlainActionFuture<GetApiKeyResponse> listener = new PlainActionFuture<>();
+        @SuppressWarnings("unchecked")
         final Tuple<String,String> invalidRealmAndUserPair = randomFrom(
             new Tuple<>("file", "user_with_run_as_role"),
             new Tuple<>("index", "user_with_manage_own_api_key_role"),
@@ -761,6 +763,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         List<CreateApiKeyResponse> userWithManageApiKeyRoleApiKeys = createApiKeys("user_with_manage_own_api_key_role",
             "user_with_run_as_role", noOfApiKeysForUserWithManageApiKeyRole, null, "monitor");
         PlainActionFuture<InvalidateApiKeyResponse> listener = new PlainActionFuture<>();
+        @SuppressWarnings("unchecked")
         final Tuple<String,String> invalidRealmAndUserPair = randomFrom(
             new Tuple<>("file", "user_with_run_as_role"),
             new Tuple<>("index", "user_with_manage_own_api_key_role"),

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java

@@ -534,6 +534,7 @@ public class TokenAuthIntegTests extends SecurityIntegTestCase {
             .setFetchSource("refresh_token", Strings.EMPTY_STRING)
             .get();
         assertNotNull(updateResponse);
+        @SuppressWarnings("unchecked")
         Map<String, Object> refreshTokenMap = (Map<String, Object>) updateResponse.getGetResult().sourceAsMap().get("refresh_token");
         assertTrue(
             Instant.ofEpochMilli((long) refreshTokenMap.get("refresh_time")).isBefore(Instant.now().minus(30L, ChronoUnit.SECONDS)));

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java

@@ -753,6 +753,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
         assertThat(usage.get("dls"), is(dls));
     }
 
+    @SuppressWarnings("unchecked")
     public void testRealmUsageStats() {
         final int numNativeUsers = scaledRandomIntBetween(1, 32);
         SecurityClient securityClient = new SecurityClient(client());

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringIntegrationTests.java

@@ -15,6 +15,7 @@ import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.xpack.core.common.socket.SocketAccess;
+import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 import org.junit.BeforeClass;
 
 import java.io.IOException;

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringUpdateTests.java

@@ -9,6 +9,7 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.SecurityIntegTestCase;
+import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 import org.junit.BeforeClass;
 
 import java.net.InetAddress;