Check cluster heath before setup-passwords (elastic/x-pack-elasticsearch#4104)

Trying to setup passwords on a red cluster (or a cluster that cannot
reach a quorum) is generally not a good idea.

This commit:
- Adds a check for RED cluster status
- Prompts to confirm execution if the cluster is red
- Prints out the reason/type is an error response is received
- Increases the HTTP read timeout so that master election failures are
  reported correctly.

Original commit: elastic/x-pack-elasticsearch@4ffbda23db

plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/CommandLineHttpClient.java

@@ -49,6 +49,14 @@ import static org.elasticsearch.xpack.core.security.SecurityField.setting;
 public class CommandLineHttpClient {
 
     public static final String HTTP_SSL_SETTING = setting("http.ssl.");
+
+    /**
+     * Timeout HTTP(s) reads after 35 seconds.
+     * The default timeout for discovering a master is 30s, and we want to be longer than this, otherwise a querying a disconnected node
+     * will trigger as client side timeout rather than giving clear error details.
+     */
+    private static final int READ_TIMEOUT = 35 * 1000;
+
     private final Settings settings;
     private final Environment env;
 
@@ -91,7 +99,7 @@ public class CommandLineHttpClient {
             conn = (HttpURLConnection) url.openConnection();
         }
         conn.setRequestMethod(method);
-        conn.setReadTimeout(30 * 1000); // 30 second timeout
+        conn.setReadTimeout(READ_TIMEOUT);
         // Add basic-auth header
         String token = UsernamePasswordToken.basicAuthHeaderValue(user, password);
         conn.setRequestProperty("Authorization", token);

plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordTool.java

@@ -35,7 +35,6 @@ import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
 import org.elasticsearch.xpack.security.authc.esnative.tool.HttpResponse.HttpResponseBuilder;
 
 import javax.net.ssl.SSLException;
-
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.HttpURLConnection;
@@ -48,6 +47,7 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static java.util.Arrays.asList;
 
@@ -80,7 +80,7 @@ public class SetupPasswordTool extends LoggingAwareMultiCommand {
     }
 
     SetupPasswordTool(CheckedFunction<Environment, CommandLineHttpClient, Exception> clientFunction,
-            CheckedFunction<Environment, KeyStoreWrapper, Exception> keyStoreFunction) {
+                      CheckedFunction<Environment, KeyStoreWrapper, Exception> keyStoreFunction) {
         super("Sets the passwords for reserved users");
         subcommands.put("auto", newAutoSetup());
         subcommands.put("interactive", newInteractiveSetup());
@@ -120,6 +120,7 @@ public class SetupPasswordTool extends LoggingAwareMultiCommand {
             terminal.println(Verbosity.VERBOSE, "Running with configuration path: " + env.configFile());
             setupOptions(options, env);
             checkElasticKeystorePasswordValid(terminal, env);
+            checkClusterHealth(terminal);
 
             if (shouldPrompt) {
                 terminal.println("Initiating the setup of passwords for reserved users " + String.join(",", USERS) + ".");
@@ -165,6 +166,7 @@ public class SetupPasswordTool extends LoggingAwareMultiCommand {
             terminal.println(Verbosity.VERBOSE, "Running with configuration path: " + env.configFile());
             setupOptions(options, env);
             checkElasticKeystorePasswordValid(terminal, env);
+            checkClusterHealth(terminal);
 
             if (shouldPrompt) {
                 terminal.println("Initiating the setup of passwords for reserved users " + String.join(",", USERS) + ".");
@@ -262,8 +264,7 @@ public class SetupPasswordTool extends LoggingAwareMultiCommand {
          * '_authenticate' call. Returns silently if server is reachable and password is
          * valid. Throws {@link UserException} otherwise.
          *
-         * @param terminal
+         * @param terminal where to write verbose info.
-         *            where to write verbose info.
          */
         void checkElasticKeystorePasswordValid(Terminal terminal, Environment env) throws Exception {
             URL route = createURL(url, "/_xpack/security/_authenticate", "?pretty");
@@ -373,13 +374,53 @@ public class SetupPasswordTool extends LoggingAwareMultiCommand {
             throw new UserException(ExitCodes.TEMP_FAILURE, "Failed to determine x-pack security feature configuration.");
         }
 
+        void checkClusterHealth(Terminal terminal) throws Exception {
+            URL route = createURL(url, "/_cluster/health", "?pretty");
+            terminal.println(Verbosity.VERBOSE, "");
+            terminal.println(Verbosity.VERBOSE, "Checking cluster health: " + route.toString());
+            final HttpResponse httpResponse = client.execute("GET", route, elasticUser, elasticUserPassword, () -> null,
+                    is -> responseBuilder(is, terminal));
+            if (httpResponse.getHttpStatus() != HttpURLConnection.HTTP_OK) {
+                terminal.println("");
+                terminal.println("Failed to determine the health of the cluster running at " + url);
+                terminal.println("Unexpected response code [" + httpResponse.getHttpStatus() + "] from calling GET " + route.toString());
+                final String cause = getErrorCause(httpResponse);
+                if (cause != null) {
+                    terminal.println("Cause: " + cause);
+                }
+            } else {
+                final String clusterStatus = Objects.toString(httpResponse.getResponseBody().get("status"), "");
+                if (clusterStatus.isEmpty()) {
+                    terminal.println("");
+                    terminal.println("Failed to determine the health of the cluster running at " + url);
+                    terminal.println("Could not find a 'status' value at " + route.toString());
+                } else if ("red".equalsIgnoreCase(clusterStatus)) {
+                    terminal.println("");
+                    terminal.println("Your cluster health is currently RED.");
+                    terminal.println("This means that some cluster data is unavailable and your cluster is not fully functional.");
+                } else {
+                    // Cluster is yellow/green -> all OK
+                    return;
+                }
+            }
+            terminal.println("");
+            terminal.println("It is recommended that you resolve the issues with your cluster before running setup-passwords.");
+            terminal.println("It is very likely that the password changes will fail when run against an unhealthy cluster.");
+            terminal.println("");
+            if (shouldPrompt) {
+                final boolean keepGoing = terminal.promptYesNo("Do you want to continue with the password setup process", false);
+                if (keepGoing == false) {
+                    throw new UserException(ExitCodes.OK, "User cancelled operation");
+                }
+                terminal.println("");
+            }
+        }
+
         /**
          * Sets one user's password using the elastic superUser credentials.
          *
-         * @param user
+         * @param user     The user who's password will change.
-         *            The user who's password will change.
+         * @param password the new password of the user.
-         * @param password
-         *            the new password of the user.
          */
         private void changeUserPassword(String user, SecureString password, Terminal terminal) throws Exception {
             URL route = createURL(url, "/_xpack/security/user/" + user + "/_password", "?pretty");
@@ -401,8 +442,14 @@ public class SetupPasswordTool extends LoggingAwareMultiCommand {
                     terminal.println("");
                     terminal.println(
                             "Unexpected response code [" + httpResponse.getHttpStatus() + "] from calling PUT " + route.toString());
+                    String cause = getErrorCause(httpResponse);
+                    if (cause != null) {
+                        terminal.println("Cause: " + cause);
+                        terminal.println("");
+                    }
                     terminal.println("Possible next steps:");
                     terminal.println("* Try running this tool again.");
+                    terminal.println("* Try running with the --verbose parameter for additional messages.");
                     terminal.println("* Check the elasticsearch logs for additional error details.");
                     terminal.println("* Use the change password API manually. ");
                     terminal.println("");
@@ -423,13 +470,11 @@ public class SetupPasswordTool extends LoggingAwareMultiCommand {
          * Collects passwords for all the users, then issues set requests. Fails on the
          * first failed request. In this case rerun the tool to redo all the operations.
          *
-         * @param passwordFn
+         * @param passwordFn      Function to generate or prompt for each user's password.
-         *            Function to generate or prompt for each user's password.
+         * @param successCallback Callback for each successful operation
-         * @param successCallback
-         *            Callback for each successful operation
          */
         void changePasswords(CheckedFunction<String, SecureString, UserException> passwordFn,
-                CheckedBiConsumer<String, SecureString, Exception> successCallback, Terminal terminal) throws Exception {
+                             CheckedBiConsumer<String, SecureString, Exception> successCallback, Terminal terminal) throws Exception {
             Map<String, SecureString> passwordsMap = new HashMap<>(USERS.size());
             try {
                 for (String user : USERS) {
@@ -473,6 +518,32 @@ public class SetupPasswordTool extends LoggingAwareMultiCommand {
         }
     }
 
+    private String getErrorCause(HttpResponse httpResponse) {
+        final Object error = httpResponse.getResponseBody().get("error");
+        if (error == null) {
+            return null;
+        }
+        if (error instanceof Map) {
+            Object reason = ((Map) error).get("reason");
+            if (reason != null) {
+                return reason.toString();
+            }
+            final Object root = ((Map) error).get("root_cause");
+            if (root != null && root instanceof Map) {
+                reason = ((Map) root).get("reason");
+                if (reason != null) {
+                    return reason.toString();
+                }
+                final Object type = ((Map) root).get("type");
+                if (type != null) {
+                    return (String) type;
+                }
+            }
+            return String.valueOf(((Map) error).get("type"));
+        }
+        return error.toString();
+    }
+
     private static URL createURL(URL url, String path, String query) throws MalformedURLException, URISyntaxException {
         URL route = new URL(url, (url.toURI().getPath() + path).replaceAll("/+", "/") + query);
         return route;
@@ -484,6 +555,7 @@ public class SetupPasswordTool extends LoggingAwareMultiCommand {
     static class XPackSecurityFeatureConfig {
         final boolean isAvailable;
         final boolean isEnabled;
+
         XPackSecurityFeatureConfig(boolean isAvailable, boolean isEnabled) {
             this.isAvailable = isAvailable;
             this.isEnabled = isEnabled;

plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordToolTests.java

@@ -12,6 +12,7 @@ import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.CheckedSupplier;
+import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.common.settings.SecureString;
@@ -35,6 +36,7 @@ import org.elasticsearch.xpack.core.security.user.ElasticUser;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
 import org.elasticsearch.xpack.security.authc.esnative.tool.HttpResponse.HttpResponseBuilder;
 import org.hamcrest.CoreMatchers;
+import org.hamcrest.Matchers;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.rules.ExpectedException;
@@ -42,6 +44,7 @@ import org.mockito.ArgumentCaptor;
 import org.mockito.InOrder;
 import org.mockito.Mockito;
 
+import javax.net.ssl.SSLException;
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
@@ -56,8 +59,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import javax.net.ssl.SSLException;
-
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Matchers.eq;
@@ -80,7 +81,7 @@ public class SetupPasswordToolTests extends CommandTestCase {
     public void setSecretsAndKeyStore() throws Exception {
         // sometimes we fall back to the keystore seed as this is the default when a new node starts
         boolean useFallback = randomBoolean();
-        bootstrapPassword = useFallback ?  new SecureString("0xCAFEBABE".toCharArray()) :
+        bootstrapPassword = useFallback ? new SecureString("0xCAFEBABE".toCharArray()) :
                 new SecureString("bootstrap-password".toCharArray());
         this.keyStore = mock(KeyStoreWrapper.class);
         this.httpClient = mock(CommandLineHttpClient.class);
@@ -102,6 +103,9 @@ public class SetupPasswordToolTests extends CommandTestCase {
                 any(CheckedFunction.class))).thenReturn(httpResponse);
 
         URL url = new URL(httpClient.getDefaultURL());
+        httpResponse = new HttpResponse(HttpURLConnection.HTTP_OK, Collections.singletonMap("status", randomFrom("yellow", "green")));
+        when(httpClient.execute(anyString(), eq(clusterHealthUrl(url)), anyString(), any(SecureString.class), any(CheckedSupplier.class),
+                any(CheckedFunction.class))).thenReturn(httpResponse);
 
         URL xpackSecurityPluginQueryURL = queryXPackSecurityFeatureConfigURL(url);
         HttpResponse queryXPackSecurityConfigHttpResponse = new HttpResponse(HttpURLConnection.HTTP_OK, new HashMap<String, Object>());
@@ -132,7 +136,7 @@ public class SetupPasswordToolTests extends CommandTestCase {
                     @Override
                     protected Environment createEnv(Map<String, String> settings) throws UserException {
                         Settings.Builder builder = Settings.builder();
-                        settings.forEach((k,v) -> builder.put(k, v));
+                        settings.forEach((k, v) -> builder.put(k, v));
                         return TestEnvironment.newEnvironment(builder.build());
                     }
                 };
@@ -144,7 +148,7 @@ public class SetupPasswordToolTests extends CommandTestCase {
                     @Override
                     protected Environment createEnv(Map<String, String> settings) throws UserException {
                         Settings.Builder builder = Settings.builder();
-                        settings.forEach((k,v) -> builder.put(k, v));
+                        settings.forEach((k, v) -> builder.put(k, v));
                         return TestEnvironment.newEnvironment(builder.build());
                     }
                 };
@@ -166,11 +170,11 @@ public class SetupPasswordToolTests extends CommandTestCase {
 
         InOrder inOrder = Mockito.inOrder(httpClient);
 
-        URL checkUrl = checkURL(url);
+        URL checkUrl = authenticateUrl(url);
         inOrder.verify(httpClient).execute(eq("GET"), eq(checkUrl), eq(ElasticUser.NAME), eq(bootstrapPassword), any(CheckedSupplier.class),
                 any(CheckedFunction.class));
         for (String user : usersInSetOrder) {
-            URL urlWithRoute = passwdURL(url, user);
+            URL urlWithRoute = passwordUrl(url, user);
             inOrder.verify(httpClient).execute(eq("PUT"), eq(urlWithRoute), eq(ElasticUser.NAME), eq(bootstrapPassword),
                     any(CheckedSupplier.class), any(CheckedFunction.class));
         }
@@ -178,7 +182,7 @@ public class SetupPasswordToolTests extends CommandTestCase {
 
     public void testAuthnFail() throws Exception {
         URL url = new URL(httpClient.getDefaultURL());
-        URL authnURL = checkURL(url);
+        URL authnURL = authenticateUrl(url);
 
         HttpResponse httpResponse = new HttpResponse(HttpURLConnection.HTTP_UNAUTHORIZED, new HashMap<String, Object>());
 
@@ -195,7 +199,7 @@ public class SetupPasswordToolTests extends CommandTestCase {
 
     public void testErrorMessagesWhenXPackIsNotAvailableOnNode() throws Exception {
         URL url = new URL(httpClient.getDefaultURL());
-        URL authnURL = checkURL(url);
+        URL authnURL = authenticateUrl(url);
 
         HttpResponse httpResponse = new HttpResponse(HttpURLConnection.HTTP_NOT_FOUND, new HashMap<String, Object>());
         when(httpClient.execute(eq("GET"), eq(authnURL), eq(ElasticUser.NAME), any(SecureString.class), any(CheckedSupplier.class),
@@ -214,7 +218,7 @@ public class SetupPasswordToolTests extends CommandTestCase {
         }
         when(httpClient.execute(eq("GET"), eq(xpackSecurityPluginQueryURL), eq(ElasticUser.NAME), any(SecureString.class),
                 any(CheckedSupplier.class), any(CheckedFunction.class)))
-                        .thenReturn(createHttpResponse(HttpURLConnection.HTTP_BAD_REQUEST, securityPluginQueryResponseBody));
+                .thenReturn(createHttpResponse(HttpURLConnection.HTTP_BAD_REQUEST, securityPluginQueryResponseBody));
 
         thrown.expect(UserException.class);
         thrown.expectMessage("X-Pack is not available on this Elasticsearch node.");
@@ -223,7 +227,7 @@ public class SetupPasswordToolTests extends CommandTestCase {
 
     public void testErrorMessagesWhenXPackIsAvailableWithCorrectLicenseAndIsEnabledButStillFailedForUnknown() throws Exception {
         URL url = new URL(httpClient.getDefaultURL());
-        URL authnURL = checkURL(url);
+        URL authnURL = authenticateUrl(url);
 
         HttpResponse httpResponse = new HttpResponse(HttpURLConnection.HTTP_NOT_FOUND, new HashMap<String, Object>());
         when(httpClient.execute(eq("GET"), eq(authnURL), eq(ElasticUser.NAME), any(SecureString.class), any(CheckedSupplier.class),
@@ -245,7 +249,7 @@ public class SetupPasswordToolTests extends CommandTestCase {
         }
         when(httpClient.execute(eq("GET"), eq(xpackSecurityPluginQueryURL), eq(ElasticUser.NAME), any(SecureString.class),
                 any(CheckedSupplier.class), any(CheckedFunction.class)))
-                        .thenReturn(createHttpResponse(HttpURLConnection.HTTP_OK, securityPluginQueryResponseBody));
+                .thenReturn(createHttpResponse(HttpURLConnection.HTTP_OK, securityPluginQueryResponseBody));
 
         thrown.expect(UserException.class);
         thrown.expectMessage("Unknown error");
@@ -255,7 +259,7 @@ public class SetupPasswordToolTests extends CommandTestCase {
 
     public void testErrorMessagesWhenXPackPluginIsAvailableButNoSecurityLicense() throws Exception {
         URL url = new URL(httpClient.getDefaultURL());
-        URL authnURL = checkURL(url);
+        URL authnURL = authenticateUrl(url);
         URL xpackSecurityPluginQueryURL = queryXPackSecurityFeatureConfigURL(url);
 
         HttpResponse httpResponse = new HttpResponse(HttpURLConnection.HTTP_NOT_FOUND, new HashMap<String, Object>());
@@ -276,7 +280,7 @@ public class SetupPasswordToolTests extends CommandTestCase {
         }
         when(httpClient.execute(eq("GET"), eq(xpackSecurityPluginQueryURL), eq(ElasticUser.NAME), any(SecureString.class),
                 any(CheckedSupplier.class), any(CheckedFunction.class)))
-                        .thenReturn(createHttpResponse(HttpURLConnection.HTTP_OK, securityPluginQueryResponseBody));
+                .thenReturn(createHttpResponse(HttpURLConnection.HTTP_OK, securityPluginQueryResponseBody));
 
         thrown.expect(UserException.class);
         thrown.expectMessage("X-Pack Security is not available.");
@@ -286,7 +290,7 @@ public class SetupPasswordToolTests extends CommandTestCase {
 
     public void testErrorMessagesWhenXPackPluginIsAvailableWithValidLicenseButDisabledSecurity() throws Exception {
         URL url = new URL(httpClient.getDefaultURL());
-        URL authnURL = checkURL(url);
+        URL authnURL = authenticateUrl(url);
         URL xpackSecurityPluginQueryURL = queryXPackSecurityFeatureConfigURL(url);
 
         HttpResponse httpResponse = new HttpResponse(HttpURLConnection.HTTP_NOT_FOUND, new HashMap<String, Object>());
@@ -307,17 +311,16 @@ public class SetupPasswordToolTests extends CommandTestCase {
         }
         when(httpClient.execute(eq("GET"), eq(xpackSecurityPluginQueryURL), eq(ElasticUser.NAME), any(SecureString.class),
                 any(CheckedSupplier.class), any(CheckedFunction.class)))
-                        .thenReturn(createHttpResponse(HttpURLConnection.HTTP_OK, securityPluginQueryResponseBody));
+                .thenReturn(createHttpResponse(HttpURLConnection.HTTP_OK, securityPluginQueryResponseBody));
 
         thrown.expect(UserException.class);
         thrown.expectMessage("X-Pack Security is disabled by configuration.");
         execute(randomBoolean() ? "auto" : "interactive", pathHomeParameter);
-
     }
 
     public void testWrongServer() throws Exception {
         URL url = new URL(httpClient.getDefaultURL());
-        URL authnURL = checkURL(url);
+        URL authnURL = authenticateUrl(url);
         doThrow(randomFrom(new IOException(), new SSLException(""))).when(httpClient).execute(eq("GET"), eq(authnURL), eq(ElasticUser.NAME),
                 any(SecureString.class), any(CheckedSupplier.class), any(CheckedFunction.class));
 
@@ -329,17 +332,39 @@ public class SetupPasswordToolTests extends CommandTestCase {
         }
     }
 
+    public void testRedCluster() throws Exception {
+        URL url = new URL(httpClient.getDefaultURL());
+
+        HttpResponse httpResponse = new HttpResponse(HttpURLConnection.HTTP_OK, new HashMap<>());
+        when(httpClient.execute(eq("GET"), eq(authenticateUrl(url)), eq(ElasticUser.NAME), any(SecureString.class),
+                any(CheckedSupplier.class), any(CheckedFunction.class))).thenReturn(httpResponse);
+
+        httpResponse = new HttpResponse(HttpURLConnection.HTTP_OK, MapBuilder.<String, Object>newMapBuilder()
+                .put("cluster_name", "elasticsearch").put("status", "red").put("number_of_nodes", 1).map());
+        when(httpClient.execute(eq("GET"), eq(clusterHealthUrl(url)), eq(ElasticUser.NAME), any(SecureString.class),
+                any(CheckedSupplier.class), any(CheckedFunction.class))).thenReturn(httpResponse);
+
+        terminal.addTextInput("n");
+        try {
+            execute(randomBoolean() ? "auto" : "interactive", pathHomeParameter);
+            fail("Should have thrown exception");
+        } catch (UserException e) {
+            assertEquals(ExitCodes.OK, e.exitCode);
+            assertThat(terminal.getOutput(), Matchers.containsString("Your cluster health is currently RED."));
+        }
+    }
+
     public void testUrlOption() throws Exception {
         URL url = new URL("http://localhost:9202" + randomFrom("", "/", "//", "/smth", "//smth/", "//x//x/"));
         execute("auto", pathHomeParameter, "-u", url.toString(), "-b");
 
         InOrder inOrder = Mockito.inOrder(httpClient);
 
-        URL checkUrl = checkURL(url);
+        URL checkUrl = authenticateUrl(url);
         inOrder.verify(httpClient).execute(eq("GET"), eq(checkUrl), eq(ElasticUser.NAME), eq(bootstrapPassword), any(CheckedSupplier.class),
                 any(CheckedFunction.class));
         for (String user : usersInSetOrder) {
-            URL urlWithRoute = passwdURL(url, user);
+            URL urlWithRoute = passwordUrl(url, user);
             inOrder.verify(httpClient).execute(eq("PUT"), eq(urlWithRoute), eq(ElasticUser.NAME), eq(bootstrapPassword),
                     any(CheckedSupplier.class), any(CheckedFunction.class));
         }
@@ -348,7 +373,7 @@ public class SetupPasswordToolTests extends CommandTestCase {
     public void testSetUserPassFail() throws Exception {
         URL url = new URL(httpClient.getDefaultURL());
         String userToFail = randomFrom(SetupPasswordTool.USERS);
-        URL userToFailURL = passwdURL(url, userToFail);
+        URL userToFailURL = passwordUrl(url, userToFail);
 
         doThrow(new IOException()).when(httpClient).execute(eq("PUT"), eq(userToFailURL), anyString(), any(SecureString.class),
                 any(CheckedSupplier.class), any(CheckedFunction.class));
@@ -368,11 +393,11 @@ public class SetupPasswordToolTests extends CommandTestCase {
 
         InOrder inOrder = Mockito.inOrder(httpClient);
 
-        URL checkUrl = checkURL(url);
+        URL checkUrl = authenticateUrl(url);
         inOrder.verify(httpClient).execute(eq("GET"), eq(checkUrl), eq(ElasticUser.NAME), eq(bootstrapPassword), any(CheckedSupplier.class),
                 any(CheckedFunction.class));
         for (String user : usersInSetOrder) {
-            URL urlWithRoute = passwdURL(url, user);
+            URL urlWithRoute = passwordUrl(url, user);
             ArgumentCaptor<CheckedSupplier<String, Exception>> passwordCaptor = ArgumentCaptor.forClass((Class) CheckedSupplier.class);
             inOrder.verify(httpClient).execute(eq("PUT"), eq(urlWithRoute), eq(ElasticUser.NAME), eq(bootstrapPassword),
                     passwordCaptor.capture(), any(CheckedFunction.class));
@@ -406,11 +431,11 @@ public class SetupPasswordToolTests extends CommandTestCase {
 
         InOrder inOrder = Mockito.inOrder(httpClient);
 
-        URL checkUrl = checkURL(url);
+        URL checkUrl = authenticateUrl(url);
         inOrder.verify(httpClient).execute(eq("GET"), eq(checkUrl), eq(ElasticUser.NAME), eq(bootstrapPassword), any(CheckedSupplier.class),
                 any(CheckedFunction.class));
         for (String user : usersInSetOrder) {
-            URL urlWithRoute = passwdURL(url, user);
+            URL urlWithRoute = passwordUrl(url, user);
             ArgumentCaptor<CheckedSupplier<String, Exception>> passwordCaptor = ArgumentCaptor.forClass((Class) CheckedSupplier.class);
             inOrder.verify(httpClient).execute(eq("PUT"), eq(urlWithRoute), eq(ElasticUser.NAME), eq(bootstrapPassword),
                     passwordCaptor.capture(), any(CheckedFunction.class));
@@ -433,14 +458,18 @@ public class SetupPasswordToolTests extends CommandTestCase {
         throw new RuntimeException("Did not properly parse password.");
     }
 
-    private URL checkURL(URL url) throws MalformedURLException, URISyntaxException {
+    private URL authenticateUrl(URL url) throws MalformedURLException, URISyntaxException {
         return new URL(url, (url.toURI().getPath() + "/_xpack/security/_authenticate").replaceAll("/+", "/") + "?pretty");
     }
 
-    private URL passwdURL(URL url, String user) throws MalformedURLException, URISyntaxException {
+    private URL passwordUrl(URL url, String user) throws MalformedURLException, URISyntaxException {
         return new URL(url, (url.toURI().getPath() + "/_xpack/security/user/" + user + "/_password").replaceAll("/+", "/") + "?pretty");
     }
 
+    private URL clusterHealthUrl(URL url) throws MalformedURLException, URISyntaxException {
+        return new URL(url, (url.toURI().getPath() + "/_cluster/health").replaceAll("/+", "/") + "?pretty");
+    }
+
     private URL queryXPackSecurityFeatureConfigURL(URL url) throws MalformedURLException, URISyntaxException {
         return new URL(url,
                 (url.toURI().getPath() + "/_xpack").replaceAll("/+", "/") + "?categories=features&human=false&pretty");