Add basic full cluster restart tests for x-pack (elastic/x-pack-elasticsearch#1743)

Adds tests similar to `:qa:full-cluster-restart` for x-pack. You
run them with `gradle :x-pack:qa:full-cluster-restart:check`.

The actual tests are as basic as it gets: create a doc and load it,
shut down, upgrade to master, startup, and load it. Create a user
and load it, shut down, upgrade to master, startup, and load it.

Relates to elastic/x-pack-elasticsearch#1629

Original commit: elastic/x-pack-elasticsearch@8994bec8e7

plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearningTemplateRegistry.java

@@ -46,7 +46,7 @@ public class MachineLearningTemplateRegistry extends AbstractComponent implement
     private final Client client;
     private final ThreadPool threadPool;
 
-    public static String [] TEMPLATE_NAMES = new String [] {Auditor.NOTIFICATIONS_INDEX, MlMetaIndex.INDEX_NAME,
+    public static final String [] TEMPLATE_NAMES = new String [] {Auditor.NOTIFICATIONS_INDEX, MlMetaIndex.INDEX_NAME,
             AnomalyDetectorsIndex.jobStateIndexName(), AnomalyDetectorsIndex.jobResultsIndexPrefix()};
 
     final AtomicBoolean putMlNotificationsIndexTemplateCheck = new AtomicBoolean(false);

plugin/src/test/java/org/elasticsearch/xpack/security/SecurityClusterClientYamlTestCase.java

@@ -32,6 +32,10 @@ public abstract class SecurityClusterClientYamlTestCase extends ESClientYamlSuit
 
     @Before
     public void waitForSecuritySetup() throws Exception {
+        waitForSecurity();
+    }
+
+    public static void waitForSecurity() throws Exception {
         String masterNode = null;
         HttpEntity entity = client().performRequest("GET", "/_cat/nodes?h=id,master").getEntity();
         String catNodesResponse = EntityUtils.toString(entity, StandardCharsets.UTF_8);
@@ -77,5 +81,4 @@ public abstract class SecurityClusterClientYamlTestCase extends ESClientYamlSuit
             }
         });
     }
-
 }

plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestCase.java

@@ -9,12 +9,23 @@ package org.elasticsearch.xpack.test.rest;
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.elasticsearch.xpack.ml.MachineLearningTemplateRegistry;
 
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static java.util.Collections.singletonMap;
 import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 
 public abstract class XPackRestTestCase extends ESClientYamlSuiteTestCase {
@@ -37,4 +48,46 @@ public abstract class XPackRestTestCase extends ESClientYamlSuiteTestCase {
                 .put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE)
                 .build();
     }
+
+    /**
+     * Waits for the Machine Learning templates to be created by {@link MachineLearningTemplateRegistry}.
+     */
+    public static void waitForMlTemplates() throws InterruptedException {
+        AtomicReference<Version> masterNodeVersion = new AtomicReference<>();
+        awaitBusy(() -> {
+            String response;
+            try {
+                response = EntityUtils
+                        .toString(client().performRequest("GET", "/_cat/nodes", singletonMap("h", "master,version")).getEntity());
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+            for (String line : response.split("\n")) {
+                if (line.startsWith("*")) {
+                    masterNodeVersion.set(Version.fromString(line.substring(2).trim()));
+                    return true;
+                }
+            }
+            return false;
+        });
+
+        for (String template : MachineLearningTemplateRegistry.TEMPLATE_NAMES) {
+            awaitBusy(() -> {
+                Map<?, ?> response;
+                try {
+                    String string = EntityUtils.toString(client().performRequest("GET", "/_template/" + template).getEntity());
+                    response = XContentHelper.convertToMap(JsonXContent.jsonXContent, string, false);
+                } catch (ResponseException e) {
+                    if (e.getResponse().getStatusLine().getStatusCode() == 404) {
+                        return false;
+                    }
+                    throw new RuntimeException(e);
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+                Map<?, ?> templateDefinition = (Map<?, ?>) response.get(template);
+                return Version.fromId((Integer) templateDefinition.get("version")).equals(masterNodeVersion.get());
+            });
+        }
+    }
 }

qa/full-cluster-restart/build.gradle

@@ -0,0 +1,218 @@
+import org.elasticsearch.gradle.test.NodeInfo
+import org.elasticsearch.gradle.test.RestIntegTestTask
+import org.elasticsearch.gradle.Version
+
+import java.nio.charset.StandardCharsets
+import java.util.regex.Matcher
+
+// Apply the java plugin to this project so the sources can be edited in an IDE
+apply plugin: 'elasticsearch.build'
+test.enabled = false
+
+dependencies {
+  testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'runtime')
+  testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'testArtifacts')
+}
+
+
+Closure waitWithAuth = { NodeInfo node, AntBuilder ant ->
+    File tmpFile = new File(node.cwd, 'wait.success')
+    // wait up to twenty seconds
+    final long stopTime = System.currentTimeMillis() + 20000L;
+    Exception lastException = null;
+    while (System.currentTimeMillis() < stopTime) {
+        lastException = null;
+        // we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned
+        HttpURLConnection httpURLConnection = null;
+        try {
+            httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=${node.config.numNodes}&wait_for_status=yellow").openConnection();
+            httpURLConnection.setRequestProperty("Authorization", "Basic " +
+                    Base64.getEncoder().encodeToString("elastic:changeme".getBytes(StandardCharsets.UTF_8)));
+            httpURLConnection.setRequestMethod("GET");
+            httpURLConnection.setConnectTimeout(1000);
+            httpURLConnection.setReadTimeout(30000); // read needs to wait for nodes!
+            httpURLConnection.connect();
+            if (httpURLConnection.getResponseCode() == 200) {
+                tmpFile.withWriter StandardCharsets.UTF_8.name(), {
+                    it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name()))
+                }
+                break;
+            }
+        } catch (Exception e) {
+            logger.debug("failed to call cluster health", e)
+            lastException = e
+        } finally {
+            if (httpURLConnection != null) {
+                httpURLConnection.disconnect();
+            }
+        }
+
+        // did not start, so wait a bit before trying again
+        Thread.sleep(500L);
+    }
+    if (tmpFile.exists() == false && lastException != null) {
+        logger.error("final attempt of calling cluster health failed", lastException)
+    }
+    return tmpFile.exists()
+}
+
+Project mainProject = project
+
+/**
+ * Subdirectories of this project are test rolling upgrades with various
+ * configuration options based on their name.
+ */
+subprojects {
+  Matcher m = project.name =~ /with(out)?-system-key/
+  if (false == m.matches()) {
+    throw new InvalidUserDataException("Invalid project name [${project.name}]")
+  }
+  boolean withSystemKey = m.group(1) == null
+
+  apply plugin: 'elasticsearch.standalone-test'
+
+  // Use resources from the rolling-upgrade project in subdirectories
+  sourceSets {
+    test {
+      java {
+        srcDirs = ["${mainProject.projectDir}/src/test/java"]
+      }
+      resources {
+        srcDirs = ["${mainProject.projectDir}/src/test/resources"]
+      }
+    }
+  }
+
+  String outputDir = "generated-resources/${project.name}"
+
+  // This is a top level task which we will add dependencies to below.
+  // It is a single task that can be used to backcompat tests against all versions.
+  task bwcTest {
+    description = 'Runs backwards compatibility tests.'
+    group = 'verification'
+  }
+
+  String output = "generated-resources/${project.name}"
+  task copyTestNodeKeystore(type: Copy) {
+    from project(':x-pack-elasticsearch:plugin')
+            .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
+    into outputDir
+  }
+
+  for (Version version : indexCompatVersions) {
+    String baseName = "v${version}"
+
+    Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) {
+      mustRunAfter(precommit)
+    }
+
+    Object extension = extensions.findByName("${baseName}#oldClusterTestCluster")
+    configure(extensions.findByName("${baseName}#oldClusterTestCluster")) {
+      dependsOn copyTestNodeKeystore
+      plugin ':x-pack-elasticsearch:plugin'
+      distribution = 'zip'
+      bwcVersion = version
+      numBwcNodes = 2
+      numNodes = 2
+      clusterName = 'full-cluster-restart'
+      waitCondition = waitWithAuth
+      setting 'xpack.security.transport.ssl.enabled', 'true'
+      setting 'xpack.ssl.keystore.path', 'testnode.jks'
+      setting 'xpack.ssl.keystore.password', 'testnode'
+      setting 'xpack.security.authc.realms.native.type', 'native'
+      setting 'xpack.security.authc.realms.native.order', '0'
+      dependsOn copyTestNodeKeystore
+      extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
+      if (withSystemKey) {
+        if (version.onOrAfter('5.1.0')) {
+          // The setting didn't exist until 5.1.0
+          setting 'xpack.security.system_key.required', 'true'
+        }
+        extraConfigFile 'x-pack/system_key',
+            "${mainProject.projectDir}/src/test/resources/system_key"
+      }
+    }
+
+    Task oldClusterTestRunner = tasks.getByName("${baseName}#oldClusterTestRunner")
+    oldClusterTestRunner.configure {
+      systemProperty 'tests.is_old_cluster', 'true'
+      systemProperty 'tests.old_cluster_version', version.toString().minus("-SNAPSHOT")
+    }
+
+    Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask)
+
+    configure(extensions.findByName("${baseName}#upgradedClusterTestCluster")) {
+      dependsOn oldClusterTestRunner,
+          "${baseName}#oldClusterTestCluster#node0.stop",
+          "${baseName}#oldClusterTestCluster#node1.stop"
+      plugin ':x-pack-elasticsearch:plugin'
+      distribution = 'zip'
+      numNodes = 2
+      clusterName = 'full-cluster-restart'
+      dataDir = { nodeNum -> oldClusterTest.nodes[nodeNum].dataDir }
+      waitCondition = waitWithAuth
+      setting 'xpack.ssl.keystore.path', 'testnode.jks'
+      setting 'xpack.ssl.keystore.password', 'testnode'
+      setting 'xpack.security.authc.realms.native.type', 'native'
+      setting 'xpack.security.authc.realms.native.order', '0'
+      dependsOn copyTestNodeKeystore
+      extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
+      if (withSystemKey) {
+        setting 'xpack.security.system_key.required', 'true'
+        extraConfigFile 'x-pack/system_key',
+            "${mainProject.projectDir}/src/test/resources/system_key"
+      }
+    }
+
+    Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner")
+    upgradedClusterTestRunner.configure {
+      systemProperty 'tests.is_old_cluster', 'false'
+      systemProperty 'tests.old_cluster_version', version.toString().minus("-SNAPSHOT")
+    }
+
+    Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") {
+      dependsOn = [upgradedClusterTest]
+    }
+
+    if (project.bwc_tests_enabled) {
+      bwcTest.dependsOn(versionBwcTest)
+    }
+  }
+
+  test.enabled = false // no unit tests for full cluster restarts, only the rest integration test
+
+  // basic integ tests includes testing bwc against the most recent version
+  task integTest {
+    if (project.bwc_tests_enabled) {
+      dependsOn = ["v${wireCompatVersions[-1]}#bwcTest"]
+    }
+  }
+  check.dependsOn(integTest)
+
+  dependencies {
+    testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'runtime')
+    testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'testArtifacts')
+  }
+
+  // copy x-pack plugin info so it is on the classpath and security manager has the right permissions
+  task copyXPackRestSpec(type: Copy) {
+    dependsOn(project.configurations.restSpec, 'processTestResources')
+    from project(':x-pack-elasticsearch:plugin').sourceSets.test.resources
+    include 'rest-api-spec/api/**'
+    into project.sourceSets.test.output.resourcesDir
+  }
+
+  task copyXPackPluginProps(type: Copy) {
+    dependsOn(copyXPackRestSpec)
+    from project(':x-pack-elasticsearch:plugin').file('src/main/plugin-metadata')
+    from project(':x-pack-elasticsearch:plugin').tasks.pluginProperties
+    into outputDir
+  }
+  project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps)
+
+  repositories {
+    maven {
+      url "https://artifacts.elastic.co/maven"
+    }
+  }
+}

qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java

@@ -0,0 +1,128 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.restart;
+
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.NotEqualMessageBuilder;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xpack.ml.MachineLearningTemplateRegistry;
+import org.elasticsearch.xpack.security.SecurityClusterClientYamlTestCase;
+import org.elasticsearch.xpack.test.rest.XPackRestTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.containsString;
+
+public class FullClusterRestartIT extends ESRestTestCase {
+    private final boolean runningAgainstOldCluster = Booleans.parseBoolean(System.getProperty("tests.is_old_cluster"));
+    private final Version oldClusterVersion = Version.fromString(System.getProperty("tests.old_cluster_version"));
+
+    @Before
+    public void waitForSecuritySetup() throws Exception {
+        SecurityClusterClientYamlTestCase.waitForSecurity();
+    }
+
+    @Before
+    public void waitForMlTemplates() throws Exception {
+        XPackRestTestCase.waitForMlTemplates();
+    }
+
+    @Override
+    protected boolean preserveIndicesUponCompletion() {
+        return true;
+    }
+
+    @Override
+    protected boolean preserveTemplatesUponCompletion() {
+        return true;
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = "Basic " + Base64.getEncoder().encodeToString("elastic:changeme".getBytes(StandardCharsets.UTF_8));
+        return Settings.builder()
+                .put(ThreadContext.PREFIX + ".Authorization", token)
+                // we increase the timeout here to 90 seconds to handle long waits for a green
+                // cluster health. the waits for green need to be longer than a minute to
+                // account for delayed shards
+                .put(ESRestTestCase.CLIENT_RETRY_TIMEOUT, "90s")
+                .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s")
+                .build();
+    }
+
+    /**
+     * Tests that a single document survives. Super basic smoke test.
+     */
+    public void testSingleDoc() throws IOException {
+        String docLocation = "/testsingledoc/doc/1";
+        String doc = "{\"test\": \"test\"}";
+
+        if (runningAgainstOldCluster) {
+            client().performRequest("PUT", docLocation, singletonMap("refresh", "true"),
+                    new StringEntity(doc, ContentType.APPLICATION_JSON));
+        }
+
+        assertThat(toStr(client().performRequest("GET", docLocation)), containsString(doc));
+    }
+
+    public void testSecurityNativeRealm() throws IOException {
+        XContentBuilder userBuilder = JsonXContent.contentBuilder().startObject();
+        userBuilder.field("password", "j@rV1s");
+        userBuilder.array("roles", "admin", "other_role1");
+        userBuilder.field("full_name", "Jack Nicholson");
+        userBuilder.field("email", "jacknich@example.com");
+        userBuilder.startObject("metadata"); {
+            userBuilder.field("intelligence", 7);
+        }
+        userBuilder.endObject();
+        userBuilder.field("enabled", true);
+        String user = userBuilder.endObject().string();
+
+        if (runningAgainstOldCluster) {
+            client().performRequest("PUT", "/_xpack/security/user/jacknich", emptyMap(),
+                    new StringEntity(user, ContentType.APPLICATION_JSON));
+        }
+
+        Map<String, Object> response = toMap(client().performRequest("GET", "/_xpack/security/user/jacknich"));
+        Map<String, Object> expected = toMap(user);
+        expected.put("username", "jacknich");
+        expected.remove("password");
+        expected = singletonMap("jacknich", expected);
+        if (false == response.equals(expected)) {
+            NotEqualMessageBuilder message = new NotEqualMessageBuilder();
+            message.compareMaps(response, expected);
+            fail("User doesn't match.\n" + message.toString());
+        }
+    }
+
+    static Map<String, Object> toMap(Response response) throws IOException {
+        return toMap(EntityUtils.toString(response.getEntity()));
+    }
+
+    static Map<String, Object> toMap(String response) throws IOException {
+        return XContentHelper.convertToMap(JsonXContent.jsonXContent, response, false);
+    }
+
+    static String toStr(Response response) throws IOException {
+        return EntityUtils.toString(response.getEntity());
+    }
+}

qa/full-cluster-restart/src/test/resources/system_key

@@ -0,0 +1 @@
+ь{Ю▌█Гю+°dTI;f└┌█⌠л╜╖l▀╥²|╞}┼jВЩУDЬvYWЪV5и┤K╢h╘8┼▀н╙P·
z~╡╫у┐a▒),$jд│.И╦шГж^▌Хw╔и╢х░38╫v ├}╞▀|╘^[ УFСь╢√█"т▒⌡г≈√╢

qa/rolling-upgrade/build.gradle

@@ -64,11 +64,11 @@ Project mainProject = project
  * configuration options based on their name.
  */
 subprojects {
-  Matcher m = project.name =~ /with(out)?-ssl-with(out)?-system-key/
+  Matcher m = project.name =~ /with(out)?-system-key/
   if (false == m.matches()) {
     throw new InvalidUserDataException("Invalid project name [${project.name}]")
   }
-  boolean withSystemKey = m.group(2) == null
+  boolean withSystemKey = m.group(1) == null
 
   apply plugin: 'elasticsearch.standalone-test'
 

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java

@@ -7,32 +7,19 @@ package org.elasticsearch.upgrades;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
-import org.apache.http.HttpStatus;
+
 import org.apache.lucene.util.TimeUnits;
-import org.elasticsearch.Version;
-import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
-import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
 import org.elasticsearch.xpack.ml.MachineLearningTemplateRegistry;
 import org.elasticsearch.xpack.security.SecurityClusterClientYamlTestCase;
+import org.elasticsearch.xpack.test.rest.XPackRestTestCase;
 import org.junit.Before;
 
-import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Base64;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Supplier;
-
-import static java.util.Collections.emptyList;
-import static java.util.Collections.singletonMap;
 
 @TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs
 public class UpgradeClusterClientYamlTestSuiteIT extends SecurityClusterClientYamlTestCase {
@@ -41,26 +28,8 @@ public class UpgradeClusterClientYamlTestSuiteIT extends SecurityClusterClientYa
      * Waits for the Machine Learning templates to be created by {@link MachineLearningTemplateRegistry}
      */
     @Before
-    @SuppressWarnings("unchecked")
     public void waitForTemplates() throws Exception {
-        List<String> templates = new ArrayList<>();
-
-        Version masterNodeVersion = findMasterVersion();
-
-        templates.addAll(Arrays.asList(MachineLearningTemplateRegistry.TEMPLATE_NAMES));
-
-        for (String template : templates) {
-            awaitCallApi("indices.get_template", singletonMap("name", template), emptyList(),
-                    response -> {
-                        // We recreate the templates for every new version, so only accept
-                        // templates that correspond to the current master node version
-                        Map<String, Object> responseBody = (Map<String, Object>) response.getBody();
-                        Map<String, Object> templateDefinition = (Map<String, Object>) responseBody.get(template);
-                        Version templateVersion = Version.fromId((Integer) templateDefinition.get("version"));
-                        return masterNodeVersion.equals(templateVersion);
-                    },
-                    () -> "Exception when waiting for [" + template + "] template to be created");
-        }
+        XPackRestTestCase.waitForMlTemplates();
     }
 
     @Override
@@ -94,50 +63,4 @@ public class UpgradeClusterClientYamlTestSuiteIT extends SecurityClusterClientYa
                 .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s")
                 .build();
     }
-
-    /**
-     * Executes an API call using the admin context, waiting for it to succeed.
-     */
-    private void awaitCallApi(String apiName,
-                              Map<String, String> params,
-                              List<Map<String, Object>> bodies,
-                              CheckedFunction<ClientYamlTestResponse, Boolean, IOException> success,
-                              Supplier<String> error) throws Exception {
-
-        AtomicReference<IOException> exceptionHolder = new AtomicReference<>();
-        awaitBusy(() -> {
-            try {
-                ClientYamlTestResponse response = getAdminExecutionContext().callApi(apiName, params, bodies, Collections.emptyMap());
-                if (response.getStatusCode() == HttpStatus.SC_OK) {
-                    exceptionHolder.set(null);
-                    return success.apply(response);
-                }
-                return false;
-            } catch (IOException e) {
-                exceptionHolder.set(e);
-            }
-            return false;
-        });
-
-        IOException exception = exceptionHolder.get();
-        if (exception != null) {
-            throw new IllegalStateException(error.get(), exception);
-        }
-    }
-
-    private Version findMasterVersion() throws Exception {
-        AtomicReference<Version> versionHolder = new AtomicReference<>();
-        awaitCallApi("cat.nodes", singletonMap("h", "m,v"), emptyList(),
-                response -> {
-                    for (String line : response.getBodyAsString().split("\n")) {
-                        if (line.startsWith("*")) {
-                            versionHolder.set(Version.fromString(line.substring(2).trim()));
-                            return true;
-                        }
-                    }
-                    return false;
-                },
-                () -> "Exception when waiting to find master node version");
-        return versionHolder.get();
-    }
 }