Revert "Test clusters: convert x-pack qa tests (#43283)" (#43549)

This reverts commit ccaa8c33ba.

x-pack/qa/core-rest-tests-with-security/build.gradle

@@ -1,4 +1,3 @@
-apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -8,27 +7,39 @@ dependencies {
 
 integTest {
   includePackaged = true
-  runner {
-    systemProperty 'tests.rest.blacklist',
-            [
-                    'index/10_with_id/Index with ID',
-                    'indices.get_alias/10_basic/Get alias against closed indices'
-            ].join(',')
-
-    systemProperty 'tests.rest.cluster.username', System.getProperty('tests.rest.cluster.username', 'test_user')
-    systemProperty 'tests.rest.cluster.password', System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')
-  }
 }
 
-testClusters.integTest {
+integTestRunner {
-  distribution = "DEFAULT"
+  systemProperty 'tests.rest.blacklist',
+    [
+     'index/10_with_id/Index with ID',
+     'indices.get_alias/10_basic/Get alias against closed indices'
+    ].join(',')
+
+  systemProperty 'tests.rest.cluster.username', System.getProperty('tests.rest.cluster.username', 'test_user')
+  systemProperty 'tests.rest.cluster.password', System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')
+}
+
+integTestCluster {
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.watcher.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
-
+  setupCommand 'setupDummyUser',
-  user username: System.getProperty('tests.rest.cluster.username', 'test_user'),
+               'bin/elasticsearch-users',
-          password:  System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')
+               'useradd', System.getProperty('tests.rest.cluster.username', 'test_user'),
+               '-p', System.getProperty('tests.rest.cluster.password', 'x-pack-test-password'),
+               '-r', 'superuser'
+  waitCondition = { node, ant ->
+    File tmpFile = new File(node.cwd, 'wait.success')
+    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
+            dest: tmpFile.toString(),
+            username: 'test_user',
+            password: 'x-pack-test-password',
+            ignoreerrors: true,
+            retries: 10)
+    return tmpFile.exists()
+  }
 }

x-pack/qa/multi-cluster-search-security/build.gradle

@@ -1,59 +1,76 @@
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
-apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
   testCompile project(':x-pack:qa')
 }
 
-task 'remote-cluster'(type: RestIntegTestTask) {
+task remoteClusterTest(type: RestIntegTestTask) {
   mustRunAfter(precommit)
-  runner {
-    systemProperty 'tests.rest.suite', 'remote_cluster'
-  }
 }
 
-testClusters.'remote-cluster' {
+remoteClusterTestCluster {
-  distribution = "DEFAULT"
+  numNodes = 2
-  numberOfNodes = 2
+  clusterName = 'remote-cluster'
-  setting 'cluster.remote.connect', "false"
+  setting 'cluster.remote.connect', false
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.watcher.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
-
+  setupCommand 'setupDummyUser',
-  user username: "test_user", password: "x-pack-test-password"
+          'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
-}
+  waitCondition = { node, ant ->
-
+    File tmpFile = new File(node.cwd, 'wait.success')
-task 'mixed-cluster'(type: RestIntegTestTask) {
+    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-  dependsOn 'remote-cluster'
+            dest: tmpFile.toString(),
-  runner {
+            username: 'test_user',
-    useCluster testClusters.'remote-cluster'
+            password: 'x-pack-test-password',
-    systemProperty 'tests.rest.suite', 'multi_cluster'
+            ignoreerrors: true,
+            retries: 10)
+    return tmpFile.exists()
   }
 }
 
-testClusters.'mixed-cluster' {
+remoteClusterTestRunner {
-  distribution = "DEFAULT"
+  systemProperty 'tests.rest.suite', 'remote_cluster'
+}
+
+task mixedClusterTest(type: RestIntegTestTask) {}
+
+mixedClusterTestCluster {
+  dependsOn remoteClusterTestRunner
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.watcher.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
-  setting 'cluster.remote.my_remote_cluster.seeds', {
+  setupCommand 'setupDummyUser',
-    testClusters.'remote-cluster'.getAllTransportPortURI().collect { "\"$it\"" }.toString()
+          'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
+  waitCondition = { node, ant ->
+    File tmpFile = new File(node.cwd, 'wait.success')
+    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
+            dest: tmpFile.toString(),
+            username: 'test_user',
+            password: 'x-pack-test-password',
+            ignoreerrors: true,
+            retries: 10)
+    return tmpFile.exists()
   }
-  setting 'cluster.remote.connections_per_cluster', "1"
+  setting 'cluster.remote.my_remote_cluster.seeds', "\"${-> remoteClusterTest.nodes.get(0).transportUri()}\""
-  setting 'cluster.remote.connect', "true"
+  setting 'cluster.remote.connections_per_cluster', 1
+  setting 'cluster.remote.connect', true
+}
 
-  user username: "test_user", password: "x-pack-test-password"
+mixedClusterTestRunner {
+  systemProperty 'tests.rest.suite', 'multi_cluster'
+  finalizedBy 'remoteClusterTestCluster#node0.stop','remoteClusterTestCluster#node1.stop'
 }
 
 task integTest {
-  dependsOn 'mixed-cluster'
+  dependsOn = [mixedClusterTest]
 }
 
 test.enabled = false // no unit tests for multi-cluster-search, only the rest integration test

x-pack/qa/multi-node/build.gradle

@@ -1,4 +1,3 @@
-apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -6,16 +5,26 @@ dependencies {
   testCompile project(':x-pack:qa')
 }
 
-testClusters.integTest {
+integTestCluster {
-  distribution = 'DEFAULT'
+  numNodes = 2
-  numberOfNodes = 2
+  clusterName = 'multi-node'
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.watcher.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
-  extraConfigFile 'roles.yml', file('roles.yml')
+  extraConfigFile 'roles.yml', 'roles.yml'
-  user username: "test-user", password: "x-pack-test-password", role: "test"
+  setupCommand 'setup-test-user', 'bin/elasticsearch-users', 'useradd', 'test-user', '-p', 'x-pack-test-password', '-r', 'test'
-  user username: "super-user", password: "x-pack-super-password"
+  setupCommand 'setup-super-user', 'bin/elasticsearch-users', 'useradd', 'super-user', '-p', 'x-pack-super-password', '-r', 'superuser'
+  waitCondition = { node, ant ->
+    File tmpFile = new File(node.cwd, 'wait.success')
+    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
+            dest: tmpFile.toString(),
+            username: 'super-user',
+            password: 'x-pack-super-password',
+            ignoreerrors: true,
+            retries: 10)
+    return tmpFile.exists()
+  }
 }

x-pack/qa/oidc-op-tests/build.gradle

@@ -1,6 +1,5 @@
 Project idpFixtureProject = xpackProject("test:idp-fixture")
 
-apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
@@ -22,12 +21,8 @@ task setupPorts {
     }
 }
 
-integTest.runner {
+integTestCluster {
-  dependsOn setupPorts
+    dependsOn setupPorts
-}
-
-testClusters.integTest {
-    distribution = 'DEFAULT'
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.security.http.ssl.enabled', 'false'
@@ -37,13 +32,13 @@ testClusters.integTest {
     // OpenID Connect Realm 1 configured for authorization grant flow
     setting 'xpack.security.authc.realms.oidc.c2id.order', '2'
     setting 'xpack.security.authc.realms.oidc.c2id.op.issuer', 'http://localhost:8080'
-    setting 'xpack.security.authc.realms.oidc.c2id.op.authorization_endpoint', { "http://127.0.0.1:${ephemeralPort}/c2id-login" }
+    setting 'xpack.security.authc.realms.oidc.c2id.op.authorization_endpoint', "http://127.0.0.1:${-> ephemeralPort}/c2id-login"
-    setting 'xpack.security.authc.realms.oidc.c2id.op.token_endpoint', { "http://127.0.0.1:${ephemeralPort}/c2id/token" }
+    setting 'xpack.security.authc.realms.oidc.c2id.op.token_endpoint', "http://127.0.0.1:${-> ephemeralPort}/c2id/token"
-    setting 'xpack.security.authc.realms.oidc.c2id.op.userinfo_endpoint', { "http://127.0.0.1:${ephemeralPort}/c2id/userinfo" }
+    setting 'xpack.security.authc.realms.oidc.c2id.op.userinfo_endpoint', "http://127.0.0.1:${-> ephemeralPort}/c2id/userinfo"
     setting 'xpack.security.authc.realms.oidc.c2id.op.jwkset_path', 'op-jwks.json'
     setting 'xpack.security.authc.realms.oidc.c2id.rp.redirect_uri', 'https://my.fantastic.rp/cb'
     setting 'xpack.security.authc.realms.oidc.c2id.rp.client_id', 'elasticsearch-rp'
-    keystore 'xpack.security.authc.realms.oidc.c2id.rp.client_secret', 'b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2'
+    keystoreSetting 'xpack.security.authc.realms.oidc.c2id.rp.client_secret', 'b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2'
     setting 'xpack.security.authc.realms.oidc.c2id.rp.response_type', 'code'
     setting 'xpack.security.authc.realms.oidc.c2id.claims.principal', 'sub'
     setting 'xpack.security.authc.realms.oidc.c2id.claims.name', 'name'
@@ -52,13 +47,13 @@ testClusters.integTest {
     // OpenID Connect Realm 2 configured for implicit flow
     setting 'xpack.security.authc.realms.oidc.c2id-implicit.order', '3'
     setting 'xpack.security.authc.realms.oidc.c2id-implicit.op.issuer', 'http://localhost:8080'
-    setting 'xpack.security.authc.realms.oidc.c2id-implicit.op.authorization_endpoint', { "http://127.0.0.1:${ephemeralPort}/c2id-login" }
+    setting 'xpack.security.authc.realms.oidc.c2id-implicit.op.authorization_endpoint', "http://127.0.0.1:${-> ephemeralPort}/c2id-login"
-    setting 'xpack.security.authc.realms.oidc.c2id-implicit.op.token_endpoint', { "http://127.0.0.1:${ephemeralPort}/c2id/token" }
+    setting 'xpack.security.authc.realms.oidc.c2id-implicit.op.token_endpoint', "http://127.0.0.1:${-> ephemeralPort}/c2id/token"
-    setting 'xpack.security.authc.realms.oidc.c2id-implicit.op.userinfo_endpoint', { "http://127.0.0.1:${ephemeralPort}/c2id/userinfo" }
+    setting 'xpack.security.authc.realms.oidc.c2id-implicit.op.userinfo_endpoint', "http://127.0.0.1:${-> ephemeralPort}/c2id/userinfo"
     setting 'xpack.security.authc.realms.oidc.c2id-implicit.op.jwkset_path', 'op-jwks.json'
     setting 'xpack.security.authc.realms.oidc.c2id-implicit.rp.redirect_uri', 'https://my.fantastic.rp/cb'
     setting 'xpack.security.authc.realms.oidc.c2id-implicit.rp.client_id', 'elasticsearch-rp'
-    keystore 'xpack.security.authc.realms.oidc.c2id-implicit.rp.client_secret', 'b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2'
+    keystoreSetting 'xpack.security.authc.realms.oidc.c2id-implicit.rp.client_secret', 'b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2'
     setting 'xpack.security.authc.realms.oidc.c2id-implicit.rp.response_type', 'id_token token'
     setting 'xpack.security.authc.realms.oidc.c2id-implicit.claims.principal', 'sub'
     setting 'xpack.security.authc.realms.oidc.c2id-implicit.claims.name', 'name'
@@ -68,7 +63,19 @@ testClusters.integTest {
     
     extraConfigFile 'op-jwks.json', idpFixtureProject.file("oidc/op-jwks.json")
 
-    user username: "test_admin", password: "x-pack-test-password"
+    setupCommand 'setupTestAdmin',
+            'bin/elasticsearch-users', 'useradd', "test_admin", '-p', 'x-pack-test-password', '-r', "superuser"
+
+    waitCondition = { node, ant ->
+        File tmpFile = new File(node.cwd, 'wait.success')
+        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
+                dest: tmpFile.toString(),
+                username: 'test_admin',
+                password: 'x-pack-test-password',
+                ignoreerrors: true,
+                retries: 10)
+        return tmpFile.exists()
+    }
 }
 
 thirdPartyAudit.enabled = false

x-pack/qa/saml-idp-tests/build.gradle

@@ -1,6 +1,6 @@
 Project idpFixtureProject = xpackProject("test:idp-fixture")
+evaluationDependsOn(idpFixtureProject.path)
 
-apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
@@ -21,12 +21,10 @@ project.sourceSets.test.output.dir(outputDir, builtBy: copyIdpFiles)
 
 task setupPorts {
     dependsOn copyIdpFiles, idpFixtureProject.postProcessFixture
-    // Don't attempt to get ephemeral ports when Docker is not available
-    onlyIf { idpFixtureProject.postProcessFixture.enabled }
     doLast {
         String portString = idpFixtureProject.postProcessFixture.ext."test.fixtures.shibboleth-idp.tcp.4443"
         int ephemeralPort = Integer.valueOf(portString)
-        File idpMetaFile = file("$outputDir/idp-metadata.xml")
+        File idpMetaFile = file(outputDir + '/idp-metadata.xml')
         List<String> lines = idpMetaFile.readLines("UTF-8")
         StringBuilder content = new StringBuilder()
         for (String line : lines) {
@@ -37,44 +35,56 @@ task setupPorts {
         idpMetaFile.write(content.toString(), "UTF-8")
     }
 }
+// Don't attempt to get ephemeral ports when Docker is not available
+setupPorts.onlyIf { idpFixtureProject.postProcessFixture.enabled }
 
+integTestCluster.dependsOn setupPorts
 
-integTest.runner.dependsOn setupPorts
+integTestCluster {
+  setting 'xpack.license.self_generated.type', 'trial'
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.security.http.ssl.enabled', 'false'
+  setting 'xpack.security.authc.token.enabled', 'true'
+  setting 'xpack.security.authc.realms.file.file.order', '0'
+  // SAML realm 1 (no authorization_realms)
+  setting 'xpack.security.authc.realms.saml.shibboleth.order', '1'
+  setting 'xpack.security.authc.realms.saml.shibboleth.idp.entity_id', 'https://test.shibboleth.elastic.local/'
+  setting 'xpack.security.authc.realms.saml.shibboleth.idp.metadata.path', 'idp-metadata.xml'
+  setting 'xpack.security.authc.realms.saml.shibboleth.sp.entity_id', 'http://mock1.http.elastic.local/'
+  // The port in the ACS URL is fake - the test will bind the mock webserver
+  // to a random port and then whenever it needs to connect to a URL on the
+  // mock webserver it will replace 54321 with the real port
+  setting 'xpack.security.authc.realms.saml.shibboleth.sp.acs', 'http://localhost:54321/saml/acs1'
+  setting 'xpack.security.authc.realms.saml.shibboleth.attributes.principal', 'uid'
+  setting 'xpack.security.authc.realms.saml.shibboleth.attributes.name', 'urn:oid:2.5.4.3'
+  // SAML realm 2 (uses authorization_realms)
+  setting 'xpack.security.authc.realms.saml.shibboleth_native.order', '2'
+  setting 'xpack.security.authc.realms.saml.shibboleth_native.idp.entity_id', 'https://test.shibboleth.elastic.local/'
+  setting 'xpack.security.authc.realms.saml.shibboleth_native.idp.metadata.path', 'idp-metadata.xml'
+  setting 'xpack.security.authc.realms.saml.shibboleth_native.sp.entity_id', 'http://mock2.http.elastic.local/'
+  setting 'xpack.security.authc.realms.saml.shibboleth_native.sp.acs', 'http://localhost:54321/saml/acs2'
+  setting 'xpack.security.authc.realms.saml.shibboleth_native.attributes.principal', 'uid'
+  setting 'xpack.security.authc.realms.saml.shibboleth_native.authorization_realms', 'native'
+  setting 'xpack.security.authc.realms.native.native.order', '3'
 
-testClusters.integTest {
+  setting 'xpack.ml.enabled', 'false'
-    distribution = "DEFAULT"
+  setting 'logger.org.elasticsearch.xpack.security', 'TRACE'
-    setting 'xpack.license.self_generated.type', 'trial'
-    setting 'xpack.security.enabled', 'true'
-    setting 'xpack.security.http.ssl.enabled', 'false'
-    setting 'xpack.security.authc.token.enabled', 'true'
-    setting 'xpack.security.authc.realms.file.file.order', '0'
-    // SAML realm 1 (no authorization_realms)
-    setting 'xpack.security.authc.realms.saml.shibboleth.order', '1'
-    setting 'xpack.security.authc.realms.saml.shibboleth.idp.entity_id', 'https://test.shibboleth.elastic.local/'
-    setting 'xpack.security.authc.realms.saml.shibboleth.idp.metadata.path', 'idp-metadata.xml'
-    setting 'xpack.security.authc.realms.saml.shibboleth.sp.entity_id', 'http://mock1.http.elastic.local/'
-    // The port in the ACS URL is fake - the test will bind the mock webserver
-    // to a random port and then whenever it needs to connect to a URL on the
-    // mock webserver it will replace 54321 with the real port
-    setting 'xpack.security.authc.realms.saml.shibboleth.sp.acs', 'http://localhost:54321/saml/acs1'
-    setting 'xpack.security.authc.realms.saml.shibboleth.attributes.principal', 'uid'
-    setting 'xpack.security.authc.realms.saml.shibboleth.attributes.name', 'urn:oid:2.5.4.3'
-    // SAML realm 2 (uses authorization_realms)
-    setting 'xpack.security.authc.realms.saml.shibboleth_native.order', '2'
-    setting 'xpack.security.authc.realms.saml.shibboleth_native.idp.entity_id', 'https://test.shibboleth.elastic.local/'
-    setting 'xpack.security.authc.realms.saml.shibboleth_native.idp.metadata.path', 'idp-metadata.xml'
-    setting 'xpack.security.authc.realms.saml.shibboleth_native.sp.entity_id', 'http://mock2.http.elastic.local/'
-    setting 'xpack.security.authc.realms.saml.shibboleth_native.sp.acs', 'http://localhost:54321/saml/acs2'
-    setting 'xpack.security.authc.realms.saml.shibboleth_native.attributes.principal', 'uid'
-    setting 'xpack.security.authc.realms.saml.shibboleth_native.authorization_realms', 'native'
-    setting 'xpack.security.authc.realms.native.native.order', '3'
 
-    setting 'xpack.ml.enabled', 'false'
+  extraConfigFile 'idp-metadata.xml', file(outputDir + "/idp-metadata.xml")
-    setting 'logger.org.elasticsearch.xpack.security', 'TRACE'
 
-    extraConfigFile 'idp-metadata.xml', file(outputDir + "/idp-metadata.xml")
+  setupCommand 'setupTestAdmin',
+            'bin/elasticsearch-users', 'useradd', "test_admin", '-p', 'x-pack-test-password', '-r', "superuser"
 
-    user username: "test_admin", password: 'x-pack-test-password'
+  waitCondition = { node, ant ->
+        File tmpFile = new File(node.cwd, 'wait.success')
+        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
+                dest: tmpFile.toString(),
+                username: 'test_admin',
+                password: 'x-pack-test-password',
+                ignoreerrors: true,
+                retries: 10)
+        return tmpFile.exists()
+    }
 }
 
 forbiddenPatterns {

x-pack/qa/security-example-spi-extension/build.gradle

@@ -1,4 +1,3 @@
-apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.esplugin'
 
 esplugin {
@@ -14,16 +13,12 @@ dependencies {
 }
 
 
-integTest.runner {
+integTestRunner {
-    dependsOn buildZip
+    systemProperty 'tests.security.manager', 'false'
 }
 
-testClusters.integTest {
+integTestCluster {
-  // This is important, so that all the modules are available too.
+  dependsOn buildZip
-  // There are index templates that use token filters that are in analysis-module and
-  // processors are being used that are in ingest-common module.
-  distribution = 'default'
-
   setting 'xpack.security.authc.realms.custom.custom.order', '0'
   setting 'xpack.security.authc.realms.custom.custom.filtered_setting', 'should be filtered'
   setting 'xpack.security.authc.realms.file.esusers.order', '1'
@@ -34,5 +29,22 @@ testClusters.integTest {
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
 
-  user  username: "test_user", password: "x-pack-test-password"
+  // This is important, so that all the modules are available too.
+  // There are index templates that use token filters that are in analysis-module and
+  // processors are being used that are in ingest-common module.
+  distribution = 'default'
+
+  setupCommand 'setupDummyUser',
+               'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
+  waitCondition = { node, ant ->
+    File tmpFile = new File(node.cwd, 'wait.success')
+    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
+            dest: tmpFile.toString(),
+            username: 'test_user',
+            password: 'x-pack-test-password',
+            ignoreerrors: true,
+            retries: 10)
+    return tmpFile.exists()
+  }
 }
+check.dependsOn integTest

x-pack/qa/security-setup-password-tests/build.gradle

@@ -1,4 +1,3 @@
-apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -8,15 +7,24 @@ dependencies {
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
-integTest.runner {
+integTestRunner {
-    nonInputProperties.systemProperty 'tests.config.dir', "${-> testClusters.integTest.singleNode().getConfigDir()}"
     systemProperty 'tests.security.manager', 'false'
+    // TODO add  tests.config.dir = {cluster.singleNode().getConfigDir()} when converting to testclusters
 }
 
-testClusters.integTest {
+integTestCluster {
-    distribution = "DEFAULT"
+    setupCommand 'setupTestAdmin',
+            'bin/elasticsearch-users', 'useradd', "test_admin", '-p', 'x-pack-test-password', '-r', "superuser"
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.license.self_generated.type', 'trial'
-
+    waitCondition = { node, ant ->
-    user username: "test_admin", password: "x-pack-test-password"
+        File tmpFile = new File(node.cwd, 'wait.success')
+        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
+                dest: tmpFile.toString(),
+                username: 'test_admin',
+                password: 'x-pack-test-password',
+                ignoreerrors: true,
+                retries: 10)
+        return tmpFile.exists()
+    }
 }

x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle

@@ -1,4 +1,3 @@
-apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -6,12 +5,21 @@ dependencies {
   testCompile project(':x-pack:qa')
 }
 
-testClusters.integTest {
+integTestCluster {
-  distribution = "DEFAULT"
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'true'
   setting 'xpack.watcher.enabled', 'true'
   setting 'xpack.security.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
+
+  // exporter settings are configured dynamically in our tests
+  // configure a local exporter, the HTTP exporter is configured via dynamic settings change
+  //setting 'xpack.monitoring.exporters.my_local.type', 'local'
+  //setting 'xpack.monitoring.exporters.my_local.index.name.time_format', 'YYYY'
+  //setting 'xpack.monitoring.exporters.my_http.type', 'http'
+  //setting 'xpack.monitoring.exporters.my_http.host', 'http'
+  //setting 'xpack.monitoring.exporters.my_http.index.name.time_format', 'YYYY-MM'
+  // one of the exporters should configure cluster alerts
+  // setting 'xpack.monitoring.exporters.my_http.cluster_alerts.management.enabled', 'true'
 }

x-pack/qa/smoke-test-security-with-mustache/build.gradle

@@ -1,4 +1,3 @@
-apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -6,11 +5,21 @@ dependencies {
   testCompile project(':x-pack:qa')
 }
 
-testClusters.integTest {
+integTestCluster {
-  distribution = "DEFAULT"
   setting 'xpack.watcher.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.license.self_generated.type', 'trial'
-  user username: "test_admin", password: "x-pack-test-password"
+  setupCommand 'setupDummyUser',
+          'bin/elasticsearch-users', 'useradd', 'test_admin', '-p', 'x-pack-test-password', '-r', 'superuser'
+  waitCondition = { node, ant ->
+    File tmpFile = new File(node.cwd, 'wait.success')
+    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
+            dest: tmpFile.toString(),
+            username: 'test_admin',
+            password: 'x-pack-test-password',
+            ignoreerrors: true,
+            retries: 10)
+    return tmpFile.exists()
+  }
 }

x-pack/qa/smoke-test-watcher-with-security/build.gradle

@@ -1,4 +1,3 @@
-apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -13,9 +12,8 @@ task copyWatcherRestTests(type: Copy) {
   include 'rest-api-spec/test/watcher/**'
 }
 
-integTest.runner.dependsOn copyWatcherRestTests
+integTestCluster {
-testClusters.integTest {
+  dependsOn copyWatcherRestTests
-  distribution = "DEFAULT"
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
@@ -24,11 +22,25 @@ testClusters.integTest {
   setting 'xpack.notification.email.account._email.smtp.host', 'host.domain'
   setting 'xpack.notification.email.account._email.smtp.port', '587'
   setting 'xpack.notification.email.account._email.smtp.user', '_user'
-  keystore  'xpack.notification.email.account._email.smtp.secure_password', '_passwd'
+  keystoreSetting 'xpack.notification.email.account._email.smtp.secure_password', '_passwd'
   setting 'xpack.license.self_generated.type', 'trial'
-  extraConfigFile 'roles.yml', file('roles.yml')
+  extraConfigFile 'roles.yml', 'roles.yml'
-  user username: "test_admin", password: "x-pack-test-password"
+  setupCommand 'setupTestAdminUser',
-  user username: "x_pack_rest_user", password: "x-pack-test-password", role: "watcher_manager"
+               'bin/elasticsearch-users', 'useradd', 'test_admin', '-p', 'x-pack-test-password', '-r', 'superuser'
-  user username: "watcher_manager", password: "x-pack-test-password", role: "watcher_manager"
+  setupCommand 'setupXpackUserForTests',
-  user username: "powerless_user", password: "x-pack-test-password", role: "crappy_role"
+               'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'watcher_manager'
+  setupCommand 'setupWatcherManagerUser',
+               'bin/elasticsearch-users', 'useradd', 'watcher_manager', '-p', 'x-pack-test-password', '-r', 'watcher_manager'
+  setupCommand 'setupPowerlessUser',
+               'bin/elasticsearch-users', 'useradd', 'powerless_user', '-p', 'x-pack-test-password', '-r', 'crappy_role'
+  waitCondition = { node, ant ->
+    File tmpFile = new File(node.cwd, 'wait.success')
+    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
+            dest: tmpFile.toString(),
+            username: 'test_admin',
+            password: 'x-pack-test-password',
+            ignoreerrors: true,
+            retries: 10)
+    return tmpFile.exists()
+  }
 }

x-pack/qa/smoke-test-watcher/build.gradle

@@ -1,4 +1,3 @@
-apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -6,8 +5,7 @@ dependencies {
   testCompile project(':x-pack:qa')
 }
 
-testClusters.integTest {
+integTestCluster {
-  distribution = "DEFAULT"
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
@@ -15,3 +13,7 @@ testClusters.integTest {
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG'
 }
+
+integTestRunner {
+  include "**/*Tests.class"
+}