From 94930d0e84c63b6a90cd851c0b353c4ee1af1948 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Tue, 18 Jun 2019 10:10:05 +0300 Subject: [PATCH] Testclusters: convert ml qa tests (#43229) * Testclusters: convert ml qa tests This PR converts the ML tests to use testclusters. --- .../ml/qa/basic-multi-node/build.gradle | 6 ++- x-pack/plugin/ml/qa/disabled/build.gradle | 5 ++- .../ml/qa/ml-with-security/build.gradle | 35 ++++++--------- .../qa/native-multi-node-tests/build.gradle | 44 ++++++++----------- .../ml/qa/no-bootstrap-tests/build.gradle | 2 +- .../ml/qa/single-node-tests/build.gradle | 4 +- 6 files changed, 42 insertions(+), 54 deletions(-) diff --git a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle index fc27aa97d7a..9bbf34e5441 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle +++ b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle @@ -1,3 +1,4 @@ +apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' @@ -6,12 +7,13 @@ dependencies { testCompile project(path: xpackModule('ml'), configuration: 'runtime') } -integTestCluster { +testClusters.integTest { + distribution = 'DEFAULT' + numberOfNodes = 3 setting 'xpack.ilm.enabled', 'false' setting 'xpack.security.enabled', 'false' setting 'xpack.monitoring.enabled', 'false' setting 'xpack.watcher.enabled', 'false' setting 'xpack.ml.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' - numNodes = 3 } diff --git a/x-pack/plugin/ml/qa/disabled/build.gradle b/x-pack/plugin/ml/qa/disabled/build.gradle index ee49189ae1a..3e7f8b5ca7f 100644 --- a/x-pack/plugin/ml/qa/disabled/build.gradle +++ b/x-pack/plugin/ml/qa/disabled/build.gradle @@ -1,3 +1,4 @@ +apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' @@ -6,9 +7,9 @@ dependencies { testCompile project(path: xpackModule('ml'), configuration: 'runtime') } -integTestCluster { +testClusters.integTest { + distribution = 'DEFAULT' setting 'xpack.ilm.enabled', 'false' setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' - numNodes = 1 } diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index 8f74b859247..cdef10955fa 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -1,3 +1,4 @@ +apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' @@ -14,7 +15,10 @@ task copyMlRestTests(type: Copy) { include 'rest-api-spec/test/ml/**' } -integTestRunner { +integTest.runner { + dependsOn copyMlRestTests +} +integTest.runner { systemProperty 'tests.rest.blacklist', [ // Remove this test because it doesn't call an ML endpoint and we don't want // to grant extra permissions to the users used in this test suite @@ -103,27 +107,14 @@ integTestRunner { ].join(',') } -integTestCluster { - dependsOn copyMlRestTests - extraConfigFile 'roles.yml', 'roles.yml' - setupCommand 'setupTestAdminUser', - 'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser' - setupCommand 'setupMlAdminUser', - 'bin/elasticsearch-users', 'useradd', 'ml_admin', '-p', 'x-pack-test-password', '-r', 'minimal,machine_learning_admin' - setupCommand 'setupMlUserUser', - 'bin/elasticsearch-users', 'useradd', 'ml_user', '-p', 'x-pack-test-password', '-r', 'minimal,machine_learning_user' - setupCommand 'setupPowerlessUser', - 'bin/elasticsearch-users', 'useradd', 'no_ml', '-p', 'x-pack-test-password', '-r', 'minimal' + +testClusters.integTest { + distribution = 'DEFAULT' + extraConfigFile 'roles.yml', file('roles.yml') + user username: "x_pack_rest_user", password: "x-pack-test-password" + user username: "ml_admin", password: "x-pack-test-password", role: "minimal,machine_learning_admin" + user username: "ml_user", password: "x-pack-test-password", role: "minimal,machine_learning_user" + user username: "no_ml", password: "x-pack-test-password", role: "minimal" setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' - waitCondition = { node, ant -> - File tmpFile = new File(node.cwd, 'wait.success') - ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow", - dest: tmpFile.toString(), - username: 'x_pack_rest_user', - password: 'x-pack-test-password', - ignoreerrors: true, - retries: 10) - return tmpFile.exists() - } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle index c6776156c94..4d4b1b6f309 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle @@ -1,3 +1,4 @@ +apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' @@ -8,14 +9,6 @@ dependencies { testCompile project(path: xpackModule('ml'), configuration: 'testArtifacts') } -integTestRunner { - /* - * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each - * other if we allow them to set the number of available processors as it's set-once in Netty. - */ - systemProperty 'es.set.netty.runtime.available.processors', 'false' -} - // location for keys and certificates File keystoreDir = new File(project.buildDir, 'keystore') File nodeKey = file("$keystoreDir/testnode.pem") @@ -31,8 +24,21 @@ task copyKeyCerts(type: Copy) { sourceSets.test.resources.srcDir(keystoreDir) processTestResources.dependsOn(copyKeyCerts) -integTestCluster { +integTest { dependsOn copyKeyCerts + runner { + /* + * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each + * other if we allow them to set the number of available processors as it's set-once in Netty. + */ + systemProperty 'es.set.netty.runtime.available.processors', 'false' + } +} + +testClusters.integTest { + numberOfNodes = 3 + distribution = 'DEFAULT' + setting 'xpack.security.enabled', 'true' setting 'xpack.ml.enabled', 'true' setting 'xpack.watcher.enabled', 'false' @@ -47,25 +53,11 @@ integTestCluster { setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.ml.min_disk_space_off_heap', '200mb' - keystoreSetting 'bootstrap.password', 'x-pack-test-password' - keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode' + keystore 'bootstrap.password', 'x-pack-test-password' + keystore 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode' - numNodes = 3 - - setupCommand 'setupDummyUser', - 'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser' + user username: "x_pack_rest_user", password: "x-pack-test-password" extraConfigFile nodeKey.name, nodeKey extraConfigFile nodeCert.name, nodeCert - - waitCondition = { node, ant -> - File tmpFile = new File(node.cwd, 'wait.success') - ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow", - dest: tmpFile.toString(), - username: 'x_pack_rest_user', - password: 'x-pack-test-password', - ignoreerrors: true, - retries: 10) - return tmpFile.exists() - } } diff --git a/x-pack/plugin/ml/qa/no-bootstrap-tests/build.gradle b/x-pack/plugin/ml/qa/no-bootstrap-tests/build.gradle index 1908fb8e092..e1131a4fa8a 100644 --- a/x-pack/plugin/ml/qa/no-bootstrap-tests/build.gradle +++ b/x-pack/plugin/ml/qa/no-bootstrap-tests/build.gradle @@ -3,4 +3,4 @@ apply plugin: 'elasticsearch.standalone-test' dependencies { testCompile project(":x-pack:plugin:core") testCompile project(path: xpackModule('ml'), configuration: 'runtime') -} +} \ No newline at end of file diff --git a/x-pack/plugin/ml/qa/single-node-tests/build.gradle b/x-pack/plugin/ml/qa/single-node-tests/build.gradle index a51e0a3141c..036b46cb0ca 100644 --- a/x-pack/plugin/ml/qa/single-node-tests/build.gradle +++ b/x-pack/plugin/ml/qa/single-node-tests/build.gradle @@ -1,3 +1,4 @@ +apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' @@ -6,7 +7,8 @@ dependencies { testCompile project(path: xpackModule('ml'), configuration: 'runtime') } -integTestCluster { +testClusters.integTest { + distribution = 'DEFAULT' setting 'xpack.ilm.enabled', 'false' setting 'xpack.security.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial'