diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearningTemplateRegistry.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearningTemplateRegistry.java index 62d60decc5c..0f830de41ef 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearningTemplateRegistry.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearningTemplateRegistry.java @@ -46,7 +46,7 @@ public class MachineLearningTemplateRegistry extends AbstractComponent implement private final Client client; private final ThreadPool threadPool; - public static String [] TEMPLATE_NAMES = new String [] {Auditor.NOTIFICATIONS_INDEX, MlMetaIndex.INDEX_NAME, + public static final String [] TEMPLATE_NAMES = new String [] {Auditor.NOTIFICATIONS_INDEX, MlMetaIndex.INDEX_NAME, AnomalyDetectorsIndex.jobStateIndexName(), AnomalyDetectorsIndex.jobResultsIndexPrefix()}; final AtomicBoolean putMlNotificationsIndexTemplateCheck = new AtomicBoolean(false); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/SecurityClusterClientYamlTestCase.java b/plugin/src/test/java/org/elasticsearch/xpack/security/SecurityClusterClientYamlTestCase.java index 92096474e76..85be1b84dd4 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/SecurityClusterClientYamlTestCase.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/SecurityClusterClientYamlTestCase.java @@ -32,6 +32,10 @@ public abstract class SecurityClusterClientYamlTestCase extends ESClientYamlSuit @Before public void waitForSecuritySetup() throws Exception { + waitForSecurity(); + } + + public static void waitForSecurity() throws Exception { String masterNode = null; HttpEntity entity = client().performRequest("GET", "/_cat/nodes?h=id,master").getEntity(); String catNodesResponse = EntityUtils.toString(entity, StandardCharsets.UTF_8); @@ -77,5 +81,4 @@ public abstract class SecurityClusterClientYamlTestCase extends ESClientYamlSuit } }); } - } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestCase.java b/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestCase.java index eab38a4cc7c..6bdae8fee99 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestCase.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestCase.java @@ -9,12 +9,23 @@ package org.elasticsearch.xpack.test.rest; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.Version; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.xpack.ml.MachineLearningTemplateRegistry; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static java.util.Collections.singletonMap; import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; public abstract class XPackRestTestCase extends ESClientYamlSuiteTestCase { @@ -37,4 +48,46 @@ public abstract class XPackRestTestCase extends ESClientYamlSuiteTestCase { .put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE) .build(); } + + /** + * Waits for the Machine Learning templates to be created by {@link MachineLearningTemplateRegistry}. + */ + public static void waitForMlTemplates() throws InterruptedException { + AtomicReference masterNodeVersion = new AtomicReference<>(); + awaitBusy(() -> { + String response; + try { + response = EntityUtils + .toString(client().performRequest("GET", "/_cat/nodes", singletonMap("h", "master,version")).getEntity()); + } catch (IOException e) { + throw new RuntimeException(e); + } + for (String line : response.split("\n")) { + if (line.startsWith("*")) { + masterNodeVersion.set(Version.fromString(line.substring(2).trim())); + return true; + } + } + return false; + }); + + for (String template : MachineLearningTemplateRegistry.TEMPLATE_NAMES) { + awaitBusy(() -> { + Map response; + try { + String string = EntityUtils.toString(client().performRequest("GET", "/_template/" + template).getEntity()); + response = XContentHelper.convertToMap(JsonXContent.jsonXContent, string, false); + } catch (ResponseException e) { + if (e.getResponse().getStatusLine().getStatusCode() == 404) { + return false; + } + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + Map templateDefinition = (Map) response.get(template); + return Version.fromId((Integer) templateDefinition.get("version")).equals(masterNodeVersion.get()); + }); + } + } } diff --git a/qa/full-cluster-restart/build.gradle b/qa/full-cluster-restart/build.gradle new file mode 100644 index 00000000000..84addb387ab --- /dev/null +++ b/qa/full-cluster-restart/build.gradle @@ -0,0 +1,218 @@ +import org.elasticsearch.gradle.test.NodeInfo +import org.elasticsearch.gradle.test.RestIntegTestTask +import org.elasticsearch.gradle.Version + +import java.nio.charset.StandardCharsets +import java.util.regex.Matcher + +// Apply the java plugin to this project so the sources can be edited in an IDE +apply plugin: 'elasticsearch.build' +test.enabled = false + +dependencies { + testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'runtime') + testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'testArtifacts') +} + + +Closure waitWithAuth = { NodeInfo node, AntBuilder ant -> + File tmpFile = new File(node.cwd, 'wait.success') + // wait up to twenty seconds + final long stopTime = System.currentTimeMillis() + 20000L; + Exception lastException = null; + while (System.currentTimeMillis() < stopTime) { + lastException = null; + // we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned + HttpURLConnection httpURLConnection = null; + try { + httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=${node.config.numNodes}&wait_for_status=yellow").openConnection(); + httpURLConnection.setRequestProperty("Authorization", "Basic " + + Base64.getEncoder().encodeToString("elastic:changeme".getBytes(StandardCharsets.UTF_8))); + httpURLConnection.setRequestMethod("GET"); + httpURLConnection.setConnectTimeout(1000); + httpURLConnection.setReadTimeout(30000); // read needs to wait for nodes! + httpURLConnection.connect(); + if (httpURLConnection.getResponseCode() == 200) { + tmpFile.withWriter StandardCharsets.UTF_8.name(), { + it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name())) + } + break; + } + } catch (Exception e) { + logger.debug("failed to call cluster health", e) + lastException = e + } finally { + if (httpURLConnection != null) { + httpURLConnection.disconnect(); + } + } + + // did not start, so wait a bit before trying again + Thread.sleep(500L); + } + if (tmpFile.exists() == false && lastException != null) { + logger.error("final attempt of calling cluster health failed", lastException) + } + return tmpFile.exists() +} + +Project mainProject = project + +/** + * Subdirectories of this project are test rolling upgrades with various + * configuration options based on their name. + */ +subprojects { + Matcher m = project.name =~ /with(out)?-system-key/ + if (false == m.matches()) { + throw new InvalidUserDataException("Invalid project name [${project.name}]") + } + boolean withSystemKey = m.group(1) == null + + apply plugin: 'elasticsearch.standalone-test' + + // Use resources from the rolling-upgrade project in subdirectories + sourceSets { + test { + java { + srcDirs = ["${mainProject.projectDir}/src/test/java"] + } + resources { + srcDirs = ["${mainProject.projectDir}/src/test/resources"] + } + } + } + + String outputDir = "generated-resources/${project.name}" + + // This is a top level task which we will add dependencies to below. + // It is a single task that can be used to backcompat tests against all versions. + task bwcTest { + description = 'Runs backwards compatibility tests.' + group = 'verification' + } + + String output = "generated-resources/${project.name}" + task copyTestNodeKeystore(type: Copy) { + from project(':x-pack-elasticsearch:plugin') + .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks') + into outputDir + } + + for (Version version : indexCompatVersions) { + String baseName = "v${version}" + + Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) { + mustRunAfter(precommit) + } + + Object extension = extensions.findByName("${baseName}#oldClusterTestCluster") + configure(extensions.findByName("${baseName}#oldClusterTestCluster")) { + dependsOn copyTestNodeKeystore + plugin ':x-pack-elasticsearch:plugin' + distribution = 'zip' + bwcVersion = version + numBwcNodes = 2 + numNodes = 2 + clusterName = 'full-cluster-restart' + waitCondition = waitWithAuth + setting 'xpack.security.transport.ssl.enabled', 'true' + setting 'xpack.ssl.keystore.path', 'testnode.jks' + setting 'xpack.ssl.keystore.password', 'testnode' + setting 'xpack.security.authc.realms.native.type', 'native' + setting 'xpack.security.authc.realms.native.order', '0' + dependsOn copyTestNodeKeystore + extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks') + if (withSystemKey) { + if (version.onOrAfter('5.1.0')) { + // The setting didn't exist until 5.1.0 + setting 'xpack.security.system_key.required', 'true' + } + extraConfigFile 'x-pack/system_key', + "${mainProject.projectDir}/src/test/resources/system_key" + } + } + + Task oldClusterTestRunner = tasks.getByName("${baseName}#oldClusterTestRunner") + oldClusterTestRunner.configure { + systemProperty 'tests.is_old_cluster', 'true' + systemProperty 'tests.old_cluster_version', version.toString().minus("-SNAPSHOT") + } + + Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask) + + configure(extensions.findByName("${baseName}#upgradedClusterTestCluster")) { + dependsOn oldClusterTestRunner, + "${baseName}#oldClusterTestCluster#node0.stop", + "${baseName}#oldClusterTestCluster#node1.stop" + plugin ':x-pack-elasticsearch:plugin' + distribution = 'zip' + numNodes = 2 + clusterName = 'full-cluster-restart' + dataDir = { nodeNum -> oldClusterTest.nodes[nodeNum].dataDir } + waitCondition = waitWithAuth + setting 'xpack.ssl.keystore.path', 'testnode.jks' + setting 'xpack.ssl.keystore.password', 'testnode' + setting 'xpack.security.authc.realms.native.type', 'native' + setting 'xpack.security.authc.realms.native.order', '0' + dependsOn copyTestNodeKeystore + extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks') + if (withSystemKey) { + setting 'xpack.security.system_key.required', 'true' + extraConfigFile 'x-pack/system_key', + "${mainProject.projectDir}/src/test/resources/system_key" + } + } + + Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner") + upgradedClusterTestRunner.configure { + systemProperty 'tests.is_old_cluster', 'false' + systemProperty 'tests.old_cluster_version', version.toString().minus("-SNAPSHOT") + } + + Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") { + dependsOn = [upgradedClusterTest] + } + + if (project.bwc_tests_enabled) { + bwcTest.dependsOn(versionBwcTest) + } + } + + test.enabled = false // no unit tests for full cluster restarts, only the rest integration test + + // basic integ tests includes testing bwc against the most recent version + task integTest { + if (project.bwc_tests_enabled) { + dependsOn = ["v${wireCompatVersions[-1]}#bwcTest"] + } + } + check.dependsOn(integTest) + + dependencies { + testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'runtime') + testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'testArtifacts') + } + + // copy x-pack plugin info so it is on the classpath and security manager has the right permissions + task copyXPackRestSpec(type: Copy) { + dependsOn(project.configurations.restSpec, 'processTestResources') + from project(':x-pack-elasticsearch:plugin').sourceSets.test.resources + include 'rest-api-spec/api/**' + into project.sourceSets.test.output.resourcesDir + } + + task copyXPackPluginProps(type: Copy) { + dependsOn(copyXPackRestSpec) + from project(':x-pack-elasticsearch:plugin').file('src/main/plugin-metadata') + from project(':x-pack-elasticsearch:plugin').tasks.pluginProperties + into outputDir + } + project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps) + + repositories { + maven { + url "https://artifacts.elastic.co/maven" + } + } +} diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java new file mode 100644 index 00000000000..6e4ac24e963 --- /dev/null +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.restart; + +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.Version; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.Booleans; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.NotEqualMessageBuilder; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.ml.MachineLearningTemplateRegistry; +import org.elasticsearch.xpack.security.SecurityClusterClientYamlTestCase; +import org.elasticsearch.xpack.test.rest.XPackRestTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Map; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; +import static org.hamcrest.Matchers.containsString; + +public class FullClusterRestartIT extends ESRestTestCase { + private final boolean runningAgainstOldCluster = Booleans.parseBoolean(System.getProperty("tests.is_old_cluster")); + private final Version oldClusterVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); + + @Before + public void waitForSecuritySetup() throws Exception { + SecurityClusterClientYamlTestCase.waitForSecurity(); + } + + @Before + public void waitForMlTemplates() throws Exception { + XPackRestTestCase.waitForMlTemplates(); + } + + @Override + protected boolean preserveIndicesUponCompletion() { + return true; + } + + @Override + protected boolean preserveTemplatesUponCompletion() { + return true; + } + + @Override + protected Settings restClientSettings() { + String token = "Basic " + Base64.getEncoder().encodeToString("elastic:changeme".getBytes(StandardCharsets.UTF_8)); + return Settings.builder() + .put(ThreadContext.PREFIX + ".Authorization", token) + // we increase the timeout here to 90 seconds to handle long waits for a green + // cluster health. the waits for green need to be longer than a minute to + // account for delayed shards + .put(ESRestTestCase.CLIENT_RETRY_TIMEOUT, "90s") + .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s") + .build(); + } + + /** + * Tests that a single document survives. Super basic smoke test. + */ + public void testSingleDoc() throws IOException { + String docLocation = "/testsingledoc/doc/1"; + String doc = "{\"test\": \"test\"}"; + + if (runningAgainstOldCluster) { + client().performRequest("PUT", docLocation, singletonMap("refresh", "true"), + new StringEntity(doc, ContentType.APPLICATION_JSON)); + } + + assertThat(toStr(client().performRequest("GET", docLocation)), containsString(doc)); + } + + public void testSecurityNativeRealm() throws IOException { + XContentBuilder userBuilder = JsonXContent.contentBuilder().startObject(); + userBuilder.field("password", "j@rV1s"); + userBuilder.array("roles", "admin", "other_role1"); + userBuilder.field("full_name", "Jack Nicholson"); + userBuilder.field("email", "jacknich@example.com"); + userBuilder.startObject("metadata"); { + userBuilder.field("intelligence", 7); + } + userBuilder.endObject(); + userBuilder.field("enabled", true); + String user = userBuilder.endObject().string(); + + if (runningAgainstOldCluster) { + client().performRequest("PUT", "/_xpack/security/user/jacknich", emptyMap(), + new StringEntity(user, ContentType.APPLICATION_JSON)); + } + + Map response = toMap(client().performRequest("GET", "/_xpack/security/user/jacknich")); + Map expected = toMap(user); + expected.put("username", "jacknich"); + expected.remove("password"); + expected = singletonMap("jacknich", expected); + if (false == response.equals(expected)) { + NotEqualMessageBuilder message = new NotEqualMessageBuilder(); + message.compareMaps(response, expected); + fail("User doesn't match.\n" + message.toString()); + } + } + + static Map toMap(Response response) throws IOException { + return toMap(EntityUtils.toString(response.getEntity())); + } + + static Map toMap(String response) throws IOException { + return XContentHelper.convertToMap(JsonXContent.jsonXContent, response, false); + } + + static String toStr(Response response) throws IOException { + return EntityUtils.toString(response.getEntity()); + } +} diff --git a/qa/full-cluster-restart/src/test/resources/system_key b/qa/full-cluster-restart/src/test/resources/system_key new file mode 100644 index 00000000000..a72e0d6e776 --- /dev/null +++ b/qa/full-cluster-restart/src/test/resources/system_key @@ -0,0 +1 @@ +{+dTI;f̭l|}jDvYWV5Kh8ΪP z~Ճa),$j.^wɴȐ38v }|^[ F"ԑǗ \ No newline at end of file diff --git a/qa/rolling-upgrade/with-ssl-with-system-key/build.gradle b/qa/full-cluster-restart/with-system-key/build.gradle similarity index 100% rename from qa/rolling-upgrade/with-ssl-with-system-key/build.gradle rename to qa/full-cluster-restart/with-system-key/build.gradle diff --git a/qa/rolling-upgrade/with-ssl-without-system-key/build.gradle b/qa/full-cluster-restart/without-system-key/build.gradle similarity index 100% rename from qa/rolling-upgrade/with-ssl-without-system-key/build.gradle rename to qa/full-cluster-restart/without-system-key/build.gradle diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 720bbf5138f..9dc732b6d33 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -64,11 +64,11 @@ Project mainProject = project * configuration options based on their name. */ subprojects { - Matcher m = project.name =~ /with(out)?-ssl-with(out)?-system-key/ + Matcher m = project.name =~ /with(out)?-system-key/ if (false == m.matches()) { throw new InvalidUserDataException("Invalid project name [${project.name}]") } - boolean withSystemKey = m.group(2) == null + boolean withSystemKey = m.group(1) == null apply plugin: 'elasticsearch.standalone-test' diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java index 15e3c3b8e6e..ae979c0db1b 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java @@ -7,32 +7,19 @@ package org.elasticsearch.upgrades; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; -import org.apache.http.HttpStatus; + import org.apache.lucene.util.TimeUnits; -import org.elasticsearch.Version; -import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; import org.elasticsearch.xpack.ml.MachineLearningTemplateRegistry; import org.elasticsearch.xpack.security.SecurityClusterClientYamlTestCase; +import org.elasticsearch.xpack.test.rest.XPackRestTestCase; import org.junit.Before; -import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Base64; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Supplier; - -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonMap; @TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs public class UpgradeClusterClientYamlTestSuiteIT extends SecurityClusterClientYamlTestCase { @@ -41,26 +28,8 @@ public class UpgradeClusterClientYamlTestSuiteIT extends SecurityClusterClientYa * Waits for the Machine Learning templates to be created by {@link MachineLearningTemplateRegistry} */ @Before - @SuppressWarnings("unchecked") public void waitForTemplates() throws Exception { - List templates = new ArrayList<>(); - - Version masterNodeVersion = findMasterVersion(); - - templates.addAll(Arrays.asList(MachineLearningTemplateRegistry.TEMPLATE_NAMES)); - - for (String template : templates) { - awaitCallApi("indices.get_template", singletonMap("name", template), emptyList(), - response -> { - // We recreate the templates for every new version, so only accept - // templates that correspond to the current master node version - Map responseBody = (Map) response.getBody(); - Map templateDefinition = (Map) responseBody.get(template); - Version templateVersion = Version.fromId((Integer) templateDefinition.get("version")); - return masterNodeVersion.equals(templateVersion); - }, - () -> "Exception when waiting for [" + template + "] template to be created"); - } + XPackRestTestCase.waitForMlTemplates(); } @Override @@ -94,50 +63,4 @@ public class UpgradeClusterClientYamlTestSuiteIT extends SecurityClusterClientYa .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s") .build(); } - - /** - * Executes an API call using the admin context, waiting for it to succeed. - */ - private void awaitCallApi(String apiName, - Map params, - List> bodies, - CheckedFunction success, - Supplier error) throws Exception { - - AtomicReference exceptionHolder = new AtomicReference<>(); - awaitBusy(() -> { - try { - ClientYamlTestResponse response = getAdminExecutionContext().callApi(apiName, params, bodies, Collections.emptyMap()); - if (response.getStatusCode() == HttpStatus.SC_OK) { - exceptionHolder.set(null); - return success.apply(response); - } - return false; - } catch (IOException e) { - exceptionHolder.set(e); - } - return false; - }); - - IOException exception = exceptionHolder.get(); - if (exception != null) { - throw new IllegalStateException(error.get(), exception); - } - } - - private Version findMasterVersion() throws Exception { - AtomicReference versionHolder = new AtomicReference<>(); - awaitCallApi("cat.nodes", singletonMap("h", "m,v"), emptyList(), - response -> { - for (String line : response.getBodyAsString().split("\n")) { - if (line.startsWith("*")) { - versionHolder.set(Version.fromString(line.substring(2).trim())); - return true; - } - } - return false; - }, - () -> "Exception when waiting to find master node version"); - return versionHolder.get(); - } } diff --git a/qa/rolling-upgrade/with-system-key/build.gradle b/qa/rolling-upgrade/with-system-key/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/rolling-upgrade/without-system-key/build.gradle b/qa/rolling-upgrade/without-system-key/build.gradle new file mode 100644 index 00000000000..e69de29bb2d