Merge remote-tracking branch 'elastic/master' into feature/sql

Original commit: elastic/x-pack-elasticsearch@6b82e2c12e

docs/en/commands/index.asciidoc

@@ -9,10 +9,12 @@
 
 * <<certgen>>
 * <<setup-passwords>>
+* <<syskeygen>>
 * <<users-command>>
 
 --
 
 include::certgen.asciidoc[]
 include::setup-passwords.asciidoc[]
+include::syskeygen.asciidoc[]
 include::users-command.asciidoc[]

docs/en/commands/syskeygen.asciidoc

@@ -0,0 +1,50 @@
+[role="xpack"]
+[[syskeygen]]
+== syskeygen
+
+The `syskeygen` command creates a system key file in `CONFIG_DIR/x-pack`.
+
+[float]
+=== Synopsis
+
+[source,shell]
+--------------------------------------------------
+bin/x-pack/syskeygen
+[-E <KeyValuePair>] [-h, --help]
+([-s, --silent] | [-v, --verbose])
+--------------------------------------------------
+
+[float]
+=== Description
+
+The command generates a `system_key` file, which you can use to symmetrically
+encrypt sensitive data. For example, you can use this key to prevent {watcher}
+from returning and storing information that contains clear text credentials. See  {xpack-ref}/encrypting-data.html[Encrypting sensitive data in {watcher}]. 
+
+IMPORTANT: The system key is a symmetric key, so the same key must be used on
+every node in the cluster.
+
+[float]
+=== Parameters
+
+`-E <KeyValuePair>`:: Configures a setting. For example, if you have a custom
+installation of {es}, you can use this parameter to specify the `ES_PATH_CONF`
+environment variable.
+
+`-h, --help`:: Returns all of the command parameters.
+
+`-s, --silent`:: Shows minimal output.
+
+`-v, --verbose`:: Shows verbose output.
+
+
+[float]
+=== Examples
+
+The following command generates a `system_key` file in the
+default `$ES_HOME/config/x-pack` directory:
+
+[source, sh]
+--------------------------------------------------
+bin/x-pack/syskeygen
+--------------------------------------------------

docs/en/ml/getting-started.asciidoc

@@ -34,17 +34,13 @@ To follow the steps in this tutorial, you will need the following
 components of the Elastic Stack:
 
 * {es} {version}, which stores the data and the analysis results
-* {xpack} {version}, which includes the beta {ml} features for both {es} and {kib}
+* {xpack} {version}, which includes the {ml} features for both {es} and {kib}
 * {kib} {version}, which provides a helpful user interface for creating and
 viewing jobs +
 
 //ll {ml} features are available to use as an API, however this tutorial
 //will focus on using the {ml} tab in the {kib} UI.
 
-WARNING: The {xpackml} features are in beta and subject to change.
-Beta features are not subject to the same support SLA as GA features,
-and deployment in production is at your own risk.
-
 See the https://www.elastic.co/support/matrix[Elastic Support Matrix] for
 information about supported operating systems.
 

docs/en/rest-api/ml/put-datafeed.asciidoc

@@ -99,7 +99,7 @@ PUT _xpack/ml/datafeeds/datafeed-total-requests
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[skip:https://github.com/elastic/elasticsearch/pull/27183]
+// TEST[setup:server_metrics_job]
 
 When the {dfeed} is created, you receive the following results:
 [source,js]

docs/en/settings/notification-settings.asciidoc

@@ -5,10 +5,16 @@
 <titleabbrev>{watcher} Settings</titleabbrev>
 ++++
 
-You configure `xpack.notification` settings in `elasticsearch.yml` to
+You configure {watcher} settings to set up {watcher} and send notifications via
-send set up {watcher} and send notifications via <<email-notification-settings, email>>,
+<<email-notification-settings,email>>,
-<<hipchat-notification-settings, HipChat>>, <<slack-notification-settings,
+<<hipchat-notification-settings,HipChat>>,
-Slack>>, and <<pagerduty-notification-settings, PagerDuty>>.
+<<slack-notification-settings,Slack>>, and
+<<pagerduty-notification-settings, PagerDuty>>.
+
+All of these settings can be added to the `elasticsearch.yml` configuration file,
+with the exception of the secure settings, which you add to the {es} keystore.
+For more information about creating and updating the {es} keystore, see
+<<secure-settings>>.
 
 [float]
 [[general-notification-settings]]
@@ -16,6 +22,18 @@ Slack>>, and <<pagerduty-notification-settings, PagerDuty>>.
 `xpack.watcher.enabled`::
 Set to `false` to disable {watcher} on the node.
 
+`xpack.watcher.encrypt_sensitive_data` (<<secure-settings,Secure>>)::
+Set to `true` to encrypt sensitive data. If this setting is enabled, you
+must also specify the `xpack.watcher.encryption_key` setting. For more
+information, see
+{xpack-ref}/encrypting-data.html[Encrypting sensitive data in {watcher}].
+
+`xpack.watcher.encryption_key` (<<secure-settings,Secure>>)::
+Specifies the path to a file that contains a key for encrypting sensitive data.
+If `xpack.watcher.encrypt_sensitive_data` is set to `true`, this setting is
+required. For more information, see
+{xpack-ref}/encrypting-data.html[Encrypting sensitive data in {watcher}].
+
 `xpack.watcher.history.cleaner_service.enabled`::
 Set to `false` (default) to disable the cleaner service, which removes previous
 versions of {watcher} indices (for example, .watcher-history*) when it

docs/en/setup/installing-xes.asciidoc

@@ -189,15 +189,12 @@ xpack.ssl.key: certs/${node.name}/${node.name}.key <1>
 xpack.ssl.certificate: certs/${node.name}/${node.name}.crt <2>
 xpack.ssl.certificate_authorities: certs/ca/ca.crt <3>
 xpack.security.transport.ssl.enabled: true
-xpack.security.http.ssl.enabled: true <4>
 -----------------------------------------------------------
 <1> If this path does not exist on every node or the file name does not match
 the `node.name` configuration setting, you must specify the full path to the
 node key file.
 <2> Alternatively, specify the full path to the node certificate.
 <3> Alternatively specify the full path to the CA certificate.
-<4> This setting is optional. It enables SSL on the HTTP layer to ensure that
-communication between HTTP clients and the cluster is encrypted.
 --
 
 .. Start {es}.

docs/en/watcher/encrypting-data.asciidoc

@@ -0,0 +1,53 @@
+[[encrypting-data]]
+== Encrypting Sensitive Data in {watcher}
+
+Watches might have access to sensitive data such as HTTP basic authentication
+information or details about your SMTP email service. You can encrypt this
+data by generating a key and adding some secure settings on each node in your
+cluster.
+
+To encrypt sensitive data in {watcher}:
+
+. Use the {ref}/syskeygen.html[syskeygen] command to create a system key file.
+
+. Copy the `system_key` file to all of the nodes in your cluster.
++
+--
+IMPORTANT: The system key is a symmetric key, so the same key must be used on
+every node in the cluster.
+
+--
+
+. Set the
+{ref}/notification-settings.html[`xpack.watcher.encrypt_sensitive_data` setting]
+in the {ref}/secure-settings.html[{es} keystore] on each node in the cluster.
++
+--
+
+For example, run the following commands to create and update the keystore:
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore create
+bin/elasticsearch-keystore add xpack.watcher.encrypt_sensitive_data true
+----------------------------------------------------------------
+--
+
+. Set the
+{ref}/notification-settings.html[`xpack.watcher.encryption_key` setting] in the
+{ref}/secure-settings.html[{es} keystore] on each node in the cluster.
++
+--
+For example, run the following command to import the `system_key` file on
+each node:
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore add-file xpack.watcher.encryption_key <filepath>/system_key
+----------------------------------------------------------------
+--
+
+. Delete the `system_key` file on each node in the cluster.
+
+NOTE: Existing watches are not affected by these changes. Only watches that you
+create after following these steps have encryption enabled. 

docs/en/watcher/index.asciidoc

@@ -74,6 +74,8 @@ include::getting-started.asciidoc[]
 
 include::how-watcher-works.asciidoc[]
 
+include::encrypting-data.asciidoc[]
+
 include::input.asciidoc[]
 
 include::trigger.asciidoc[]

docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java

@@ -88,7 +88,6 @@ public class XDocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
                 if (state.equals("started") == false || state.equals("starting") == false) {
                     getAdminExecutionContext().callApi("xpack.watcher.start", emptyMap(), emptyList(), emptyMap());
                 }
-
                 assertThat(state, is("started"));
             });
         }
@@ -122,4 +121,9 @@ public class XDocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     public void cleanMlState() {
         new MlRestTestStateCleaner(logger, adminClient(), this);
     }
+
+    @Override
+    protected boolean randomizeContentType() {
+        return false;
+    }
 }

plugin/build.gradle

@@ -68,6 +68,9 @@ configurations {
 }
 
 dependencies {
+  // CLI deps
+  compile project(path: ':core:cli', configuration: 'runtime')
+
   // Request and Response objects
   compile "org.elasticsearch:x-pack-client-api-objects:${version}"
 

plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/CommandLineHttpClient.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.security.authc.esnative.tool;
 
-import org.bouncycastle.util.io.Streams;
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.Strings;

plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordTool.java

@@ -7,8 +7,8 @@ package org.elasticsearch.xpack.security.authc.esnative.tool;
 
 import javax.net.ssl.SSLException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.HttpURLConnection;
-import java.net.SocketException;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.security.SecureRandom;
@@ -21,6 +21,7 @@ import joptsimple.OptionParser;
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import org.bouncycastle.util.io.Streams;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.MultiCommand;
@@ -113,7 +114,7 @@ public class SetupPasswordTool extends MultiCommand {
             checkElasticKeystorePasswordValid(terminal, env);
 
             if (shouldPrompt) {
-                terminal.println("Initiating the setup of reserved user " + String.join(",", USERS) + " passwords.");
+                terminal.println("Initiating the setup of passwords for reserved users " + String.join(",", USERS) + ".");
                 terminal.println("The passwords will be randomly generated and printed to the console.");
                 boolean shouldContinue = terminal.promptYesNo("Please confirm that you would like to continue", false);
                 terminal.println("\n");
@@ -124,7 +125,7 @@ public class SetupPasswordTool extends MultiCommand {
 
             SecureRandom secureRandom = new SecureRandom();
             changePasswords((user) -> generatePassword(secureRandom, user),
-                    (user, password) -> changedPasswordCallback(terminal, user, password));
+                    (user, password) -> changedPasswordCallback(terminal, user, password), terminal);
         }
 
         private SecureString generatePassword(SecureRandom secureRandom, String user) {
@@ -158,7 +159,7 @@ public class SetupPasswordTool extends MultiCommand {
             checkElasticKeystorePasswordValid(terminal, env);
 
             if (shouldPrompt) {
-                terminal.println("Initiating the setup of reserved user " + String.join(",", USERS) + " passwords.");
+                terminal.println("Initiating the setup of passwords for reserved users " + String.join(",", USERS) + ".");
                 terminal.println("You will be prompted to enter passwords as the process progresses.");
                 boolean shouldContinue = terminal.promptYesNo("Please confirm that you would like to continue", false);
                 terminal.println("\n");
@@ -168,7 +169,7 @@ public class SetupPasswordTool extends MultiCommand {
             }
 
             changePasswords(user -> promptForPassword(terminal, user),
-                    (user, password) -> changedPasswordCallback(terminal, user, password));
+                    (user, password) -> changedPasswordCallback(terminal, user, password), terminal);
         }
 
         private SecureString promptForPassword(Terminal terminal, String user) throws UserException {
@@ -212,7 +213,7 @@ public class SetupPasswordTool extends MultiCommand {
 
         private String elasticUser = ElasticUser.NAME;
         private SecureString elasticUserPassword;
-        private String url;
+        private URL url;
 
         SetupCommand(String description) {
             super(description);
@@ -223,7 +224,7 @@ public class SetupPasswordTool extends MultiCommand {
             client = clientFunction.apply(env);
             try (KeyStoreWrapper keyStore = keyStoreFunction.apply(env)) {
                 String providedUrl = urlOption.value(options);
-                url = providedUrl == null ? client.getDefaultURL() : providedUrl;
+                url = new URL(providedUrl == null ? client.getDefaultURL() : providedUrl);
                 setShouldPrompt(options);
 
                 // TODO: We currently do not support keystore passwords
@@ -234,7 +235,7 @@ public class SetupPasswordTool extends MultiCommand {
         }
 
         private void setParser() {
-            urlOption = parser.acceptsAll(Arrays.asList("u", "url"), "The url for the change password request.").withOptionalArg();
+            urlOption = parser.acceptsAll(Arrays.asList("u", "url"), "The url for the change password request.").withRequiredArg();
             noPromptOption = parser.acceptsAll(Arrays.asList("b", "batch"),
                     "If enabled, run the change password process without prompting the user.").withOptionalArg();
         }
@@ -257,14 +258,12 @@ public class SetupPasswordTool extends MultiCommand {
          *            where to write verbose info.
          */
         void checkElasticKeystorePasswordValid(Terminal terminal, Environment env) throws Exception {
-            URL route = new URL(url + "/_xpack/security/_authenticate?pretty");
+            URL route = new URL(url, (url.toURI().getPath() + "/_xpack/security/_authenticate").replaceAll("/+", "/") + "?pretty");
+            terminal.println(Verbosity.VERBOSE, "");
+            terminal.println(Verbosity.VERBOSE, "Testing if bootstrap password is valid for " + route.toString());
             try {
-                terminal.println(Verbosity.VERBOSE, "");
+                final int httpCode = client.postURL("GET", route, elasticUser, elasticUserPassword, () -> null,
-                terminal.println(Verbosity.VERBOSE, "Testing if bootstrap password is valid for " + route.toString());
+                        is -> verboseLogResponse(is, terminal));
-                int httpCode = client.postURL("GET", route, elasticUser, elasticUserPassword, () -> null, is -> {
-                    byte[] bytes = Streams.readAll(is);
-                    terminal.println(Verbosity.VERBOSE, new String(bytes, StandardCharsets.UTF_8));
-                });
                 // keystore password is not valid
                 if (httpCode == HttpURLConnection.HTTP_UNAUTHORIZED) {
                     terminal.println("");
@@ -275,23 +274,33 @@ public class SetupPasswordTool extends MultiCommand {
                     terminal.println("   This tool used the keystore at " + KeyStoreWrapper.keystorePath(env.configFile()));
                     terminal.println("");
                     throw new UserException(ExitCodes.CONFIG, "Failed to verify bootstrap password");
+                } else if (httpCode != HttpURLConnection.HTTP_OK) {
+                    terminal.println("");
+                    terminal.println("Unexpected response code [" + httpCode + "] from calling GET " + route.toString());
+                    terminal.println("Possible causes include:");
+                    terminal.println(" * The relative path of the URL is incorrect. Is there a proxy in-between?");
+                    terminal.println(" * The protocol (http/https) does not match the port.");
+                    terminal.println(" * Is this really an Elasticsearch server?");
+                    terminal.println("");
+                    throw new UserException(ExitCodes.CONFIG, "Uknown error");
                 }
-            } catch (SocketException e) {
-                terminal.println("");
-                terminal.println("Cannot connect to elasticsearch node.");
-                e.printStackTrace(terminal.getWriter());
-                terminal.println("");
-                throw new UserException(ExitCodes.CONFIG,
-                        "Failed to connect to elasticsearch at " + route.toString() + ". Is the URL correct and elasticsearch running?", e);
             } catch (SSLException e) {
                 terminal.println("");
                 terminal.println("SSL connection to " + route.toString() + " failed: " + e.getMessage());
                 terminal.println("Please check the elasticsearch SSL settings under " + CommandLineHttpClient.HTTP_SSL_SETTING);
-                terminal.println("");
+                terminal.println(Verbosity.VERBOSE, "");
-                e.printStackTrace(terminal.getWriter());
+                terminal.println(Verbosity.VERBOSE, ExceptionsHelper.stackTrace(e));
                 terminal.println("");
                 throw new UserException(ExitCodes.CONFIG,
                         "Failed to establish SSL connection to elasticsearch at " + route.toString() + ". ", e);
+            } catch (IOException e) {
+                terminal.println("");
+                terminal.println("Connection failure to: " + route.toString() + " failed: " + e.getMessage());
+                terminal.println(Verbosity.VERBOSE, "");
+                terminal.println(Verbosity.VERBOSE, ExceptionsHelper.stackTrace(e));
+                terminal.println("");
+                throw new UserException(ExitCodes.CONFIG, "Failed to connect to elasticsearch at " +
+                        route.toString() + ". Is the URL correct and elasticsearch running?", e);
             }
         }
 
@@ -303,12 +312,15 @@ public class SetupPasswordTool extends MultiCommand {
          * @param password
          *            the new password of the user.
          */
-        private void changeUserPassword(String user, SecureString password) throws Exception {
+        private void changeUserPassword(String user, SecureString password, Terminal terminal) throws Exception {
-            URL route = new URL(url + "/_xpack/security/user/" + user + "/_password");
+            URL route = new URL(url, (url.toURI().getPath() + "/_xpack/security/user/" + user + "/_password").replaceAll("/+", "/") +
+                    "?pretty");
+            terminal.println(Verbosity.VERBOSE, "");
+            terminal.println(Verbosity.VERBOSE, "Trying user password change call " + route.toString());
             try {
                 // supplier should own his resources
                 SecureString supplierPassword = password.clone();
-                client.postURL("PUT", route, elasticUser, elasticUserPassword, () -> {
+                final int httpCode = client.postURL("PUT", route, elasticUser, elasticUserPassword, () -> {
                     try {
                         XContentBuilder xContentBuilder = JsonXContent.contentBuilder();
                         xContentBuilder.startObject().field("password", supplierPassword.toString()).endObject();
@@ -316,9 +328,24 @@ public class SetupPasswordTool extends MultiCommand {
                     } finally {
                         supplierPassword.close();
                     }
-                }, is -> {
+                }, is -> verboseLogResponse(is, terminal));
-                });
+                if (httpCode != HttpURLConnection.HTTP_OK) {
+                    terminal.println("");
+                    terminal.println("Unexpected response code [" + httpCode + "] from calling PUT " + route.toString());
+                    terminal.println("Possible next steps:");
+                    terminal.println("* Try running this tool again.");
+                    terminal.println("* Check the elasticsearch logs for additional error details.");
+                    terminal.println("* Use the change password API manually. ");
+                    terminal.println("");
+                    throw new UserException(ExitCodes.TEMP_FAILURE,
+                            "Failed to set password for user [" + user + "].");
+                }
             } catch (IOException e) {
+                terminal.println("");
+                terminal.println("Connection failure to: " + route.toString() + " failed: " + e.getMessage());
+                terminal.println(Verbosity.VERBOSE, "");
+                terminal.println(Verbosity.VERBOSE, ExceptionsHelper.stackTrace(e));
+                terminal.println("");
                 throw new UserException(ExitCodes.TEMP_FAILURE, "Failed to set password for user [" + user + "].", e);
             }
         }
@@ -333,7 +360,7 @@ public class SetupPasswordTool extends MultiCommand {
          *            Callback for each successful operation
          */
         void changePasswords(CheckedFunction<String, SecureString, UserException> passwordFn,
-                CheckedBiConsumer<String, SecureString, Exception> successCallback) throws Exception {
+                CheckedBiConsumer<String, SecureString, Exception> successCallback, Terminal terminal) throws Exception {
             Map<String, SecureString> passwordsMap = new HashMap<>(USERS.size());
             try {
                 for (String user : USERS) {
@@ -350,17 +377,27 @@ public class SetupPasswordTool extends MultiCommand {
                         superUserEntry = entry;
                         continue;
                     }
-                    changeUserPassword(entry.getKey(), entry.getValue());
+                    changeUserPassword(entry.getKey(), entry.getValue(), terminal);
                     successCallback.accept(entry.getKey(), entry.getValue());
                 }
                 // change elastic superuser
                 if (superUserEntry != null) {
-                    changeUserPassword(superUserEntry.getKey(), superUserEntry.getValue());
+                    changeUserPassword(superUserEntry.getKey(), superUserEntry.getValue(), terminal);
                     successCallback.accept(superUserEntry.getKey(), superUserEntry.getValue());
                 }
             } finally {
                 passwordsMap.forEach((user, pass) -> pass.close());
             }
         }
+
+        private void verboseLogResponse(InputStream is, Terminal terminal) throws IOException {
+            if (is != null) {
+                byte[] bytes = Streams.readAll(is);
+                terminal.println(Verbosity.VERBOSE, new String(bytes, StandardCharsets.UTF_8));
+            } else {
+                terminal.println(Verbosity.VERBOSE, "<Empty response>");
+            }
+        }
     }
+
 }

plugin/src/test/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommandTests.java

@@ -51,7 +51,7 @@ public class ListXPackExtensionCommandTests extends ESTestCase {
         }
 
         @Override
-        protected Environment createEnv(Terminal terminal, Map<String, String> settings) throws UserException {
+        protected Environment createEnv(Map<String, String> settings) throws UserException {
             return env;
         }
 

plugin/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateToolTests.java

@@ -46,7 +46,7 @@ public class ESNativeRealmMigrateToolTests extends CommandTestCase {
                 return new MigrateUserOrRoles() {
 
                     @Override
-                    protected Environment createEnv(Terminal terminal, Map<String, String> settings) throws UserException {
+                    protected Environment createEnv(Map<String, String> settings) throws UserException {
                         Settings.Builder builder = Settings.builder();
                         settings.forEach((k,v) -> builder.put(k, v));
                         return TestEnvironment.newEnvironment(builder.build());

plugin/src/test/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordToolTests.java

@@ -28,9 +28,10 @@ import org.junit.Before;
 import org.mockito.ArgumentCaptor;
 import org.mockito.InOrder;
 import org.mockito.Mockito;
-
 import java.io.IOException;
 import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -38,10 +39,11 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-
+import javax.net.ssl.SSLException;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -100,7 +102,7 @@ public class SetupPasswordToolTests extends CommandTestCase {
             protected AutoSetup newAutoSetup() {
                 return new AutoSetup() {
                     @Override
-                    protected Environment createEnv(Terminal terminal, Map<String, String> settings) throws UserException {
+                    protected Environment createEnv(Map<String, String> settings) throws UserException {
                         Settings.Builder builder = Settings.builder();
                         settings.forEach((k,v) -> builder.put(k, v));
                         return TestEnvironment.newEnvironment(builder.build());
@@ -112,7 +114,7 @@ public class SetupPasswordToolTests extends CommandTestCase {
             protected InteractiveSetup newInteractiveSetup() {
                 return new InteractiveSetup() {
                     @Override
-                    protected Environment createEnv(Terminal terminal, Map<String, String> settings) throws UserException {
+                    protected Environment createEnv(Map<String, String> settings) throws UserException {
                         Settings.Builder builder = Settings.builder();
                         settings.forEach((k,v) -> builder.put(k, v));
                         return TestEnvironment.newEnvironment(builder.build());
@@ -124,26 +126,31 @@ public class SetupPasswordToolTests extends CommandTestCase {
     }
 
     public void testAutoSetup() throws Exception {
-        String url = httpClient.getDefaultURL();
+        URL url = new URL(httpClient.getDefaultURL());
-        execute("auto", pathHomeParameter, "-b", "true");
+        if (randomBoolean()) {
+            execute("auto", pathHomeParameter, "-b", "true");
+        } else {
+            terminal.addTextInput("Y");
+            execute("auto", pathHomeParameter);
+        }
 
         verify(keyStore).decrypt(new char[0]);
 
         InOrder inOrder = Mockito.inOrder(httpClient);
 
-        URL checkUrl = new URL(url + "/_xpack/security/_authenticate?pretty");
+        URL checkUrl = checkURL(url);
         inOrder.verify(httpClient).postURL(eq("GET"), eq(checkUrl), eq(ElasticUser.NAME), eq(bootstrapPassword), any(CheckedSupplier.class),
                 any(CheckedConsumer.class));
         for (String user : usersInSetOrder) {
-            URL urlWithRoute = new URL(url + "/_xpack/security/user/" + user + "/_password");
+            URL urlWithRoute = passwdURL(url, user);
             inOrder.verify(httpClient).postURL(eq("PUT"), eq(urlWithRoute), eq(ElasticUser.NAME), eq(bootstrapPassword),
                     any(CheckedSupplier.class), any(CheckedConsumer.class));
         }
-
     }
 
     public void testAuthnFail() throws Exception {
-        URL authnURL = new URL(httpClient.getDefaultURL() + "/_xpack/security/_authenticate?pretty");
+        URL url = new URL(httpClient.getDefaultURL());
+        URL authnURL = checkURL(url);
         when(httpClient.postURL(eq("GET"), eq(authnURL), eq(ElasticUser.NAME), any(SecureString.class), any(CheckedSupplier.class),
                 any(CheckedConsumer.class))).thenReturn(HttpURLConnection.HTTP_UNAUTHORIZED);
 
@@ -153,38 +160,66 @@ public class SetupPasswordToolTests extends CommandTestCase {
         } catch (UserException e) {
             assertEquals(ExitCodes.CONFIG, e.exitCode);
         }
+    }
 
+    public void testWrongServer() throws Exception {
+        URL url = new URL(httpClient.getDefaultURL());
+        URL authnURL = checkURL(url);
+        doThrow(randomFrom(new IOException(), new SSLException(""))).when(httpClient).postURL(eq("GET"), eq(authnURL), eq(ElasticUser.NAME),
+                any(SecureString.class), any(CheckedSupplier.class), any(CheckedConsumer.class));
+
+        try {
+            execute(randomBoolean() ? "auto" : "interactive", pathHomeParameter);
+            fail("Should have thrown exception");
+        } catch (UserException e) {
+            assertEquals(ExitCodes.CONFIG, e.exitCode);
+        }
     }
 
     public void testUrlOption() throws Exception {
-        String url = "http://localhost:9202";
+        URL url = new URL("http://localhost:9202" + randomFrom("", "/", "//", "/smth", "//smth/", "//x//x/"));
-        execute("auto", pathHomeParameter, "-u", url, "-b");
+        execute("auto", pathHomeParameter, "-u", url.toString(), "-b");
 
         InOrder inOrder = Mockito.inOrder(httpClient);
 
-        URL checkUrl = new URL(url + "/_xpack/security/_authenticate?pretty");
+        URL checkUrl = checkURL(url);
         inOrder.verify(httpClient).postURL(eq("GET"), eq(checkUrl), eq(ElasticUser.NAME), eq(bootstrapPassword), any(CheckedSupplier.class),
                 any(CheckedConsumer.class));
         for (String user : usersInSetOrder) {
-            URL urlWithRoute = new URL(url + "/_xpack/security/user/" + user + "/_password");
+            URL urlWithRoute = passwdURL(url, user);
             inOrder.verify(httpClient).postURL(eq("PUT"), eq(urlWithRoute), eq(ElasticUser.NAME), eq(bootstrapPassword),
                     any(CheckedSupplier.class), any(CheckedConsumer.class));
         }
     }
 
+    public void testSetUserPassFail() throws Exception {
+        URL url = new URL(httpClient.getDefaultURL());
+        String userToFail = randomFrom(SetupPasswordTool.USERS);
+        URL userToFailURL = passwdURL(url, userToFail);
+
+        doThrow(new IOException()).when(httpClient).postURL(eq("PUT"), eq(userToFailURL), anyString(), any(SecureString.class),
+                any(CheckedSupplier.class), any(CheckedConsumer.class));
+        try {
+            execute(randomBoolean() ? "auto" : "interactive", pathHomeParameter, "-b");
+            fail("Should have thrown exception");
+        } catch (UserException e) {
+            assertEquals(ExitCodes.TEMP_FAILURE, e.exitCode);
+        }
+    }
+
     public void testInteractiveSetup() throws Exception {
-        String url = httpClient.getDefaultURL();
+        URL url = new URL(httpClient.getDefaultURL());
 
         terminal.addTextInput("Y");
         execute("interactive", pathHomeParameter);
 
         InOrder inOrder = Mockito.inOrder(httpClient);
 
-        URL checkUrl = new URL(url + "/_xpack/security/_authenticate?pretty");
+        URL checkUrl = checkURL(url);
         inOrder.verify(httpClient).postURL(eq("GET"), eq(checkUrl), eq(ElasticUser.NAME), eq(bootstrapPassword), any(CheckedSupplier.class),
                 any(CheckedConsumer.class));
         for (String user : usersInSetOrder) {
-            URL urlWithRoute = new URL(url + "/_xpack/security/user/" + user + "/_password");
+            URL urlWithRoute = passwdURL(url, user);
             ArgumentCaptor<CheckedSupplier<String, Exception>> passwordCaptor = ArgumentCaptor.forClass((Class) CheckedSupplier.class);
             inOrder.verify(httpClient).postURL(eq("PUT"), eq(urlWithRoute), eq(ElasticUser.NAME), eq(bootstrapPassword),
                     passwordCaptor.capture(), any(CheckedConsumer.class));
@@ -193,7 +228,7 @@ public class SetupPasswordToolTests extends CommandTestCase {
     }
 
     public void testInteractivePasswordsFatFingers() throws Exception {
-        String url = httpClient.getDefaultURL();
+        URL url = new URL(httpClient.getDefaultURL());
 
         terminal.reset();
         terminal.addTextInput("Y");
@@ -218,11 +253,11 @@ public class SetupPasswordToolTests extends CommandTestCase {
 
         InOrder inOrder = Mockito.inOrder(httpClient);
 
-        URL checkUrl = new URL(url + "/_xpack/security/_authenticate?pretty");
+        URL checkUrl = checkURL(url);
         inOrder.verify(httpClient).postURL(eq("GET"), eq(checkUrl), eq(ElasticUser.NAME), eq(bootstrapPassword), any(CheckedSupplier.class),
                 any(CheckedConsumer.class));
         for (String user : usersInSetOrder) {
-            URL urlWithRoute = new URL(url + "/_xpack/security/user/" + user + "/_password");
+            URL urlWithRoute = passwdURL(url, user);
             ArgumentCaptor<CheckedSupplier<String, Exception>> passwordCaptor = ArgumentCaptor.forClass((Class) CheckedSupplier.class);
             inOrder.verify(httpClient).postURL(eq("PUT"), eq(urlWithRoute), eq(ElasticUser.NAME), eq(bootstrapPassword),
                     passwordCaptor.capture(), any(CheckedConsumer.class));
@@ -243,4 +278,12 @@ public class SetupPasswordToolTests extends CommandTestCase {
         }
         throw new RuntimeException("Did not properly parse password.");
     }
+
+    private URL checkURL(URL url) throws MalformedURLException, URISyntaxException {
+        return new URL(url, (url.toURI().getPath() + "/_xpack/security/_authenticate").replaceAll("/+", "/") + "?pretty");
+    }
+
+    private URL passwdURL(URL url, String user) throws MalformedURLException, URISyntaxException {
+        return new URL(url, (url.toURI().getPath() + "/_xpack/security/user/" + user + "/_password").replaceAll("/+", "/") + "?pretty");
+    }
 }

plugin/src/test/java/org/elasticsearch/xpack/security/authc/file/tool/UsersToolTests.java

@@ -110,7 +110,7 @@ public class UsersToolTests extends CommandTestCase {
             protected AddUserCommand newAddUserCommand() {
                 return new AddUserCommand() {
                     @Override
-                    protected Environment createEnv(Terminal terminal, Map<String, String> settings) throws UserException {
+                    protected Environment createEnv(Map<String, String> settings) throws UserException {
                         return new Environment(UsersToolTests.this.settings, confDir.getParent());
                     }
                 };
@@ -120,7 +120,7 @@ public class UsersToolTests extends CommandTestCase {
             protected DeleteUserCommand newDeleteUserCommand() {
                 return new DeleteUserCommand() {
                     @Override
-                    protected Environment createEnv(Terminal terminal, Map<String, String> settings) throws UserException {
+                    protected Environment createEnv(Map<String, String> settings) throws UserException {
                         return new Environment(UsersToolTests.this.settings, confDir.getParent());
                     }
                 };
@@ -130,7 +130,7 @@ public class UsersToolTests extends CommandTestCase {
             protected PasswordCommand newPasswordCommand() {
                 return new PasswordCommand() {
                     @Override
-                    protected Environment createEnv(Terminal terminal, Map<String, String> settings) throws UserException {
+                    protected Environment createEnv(Map<String, String> settings) throws UserException {
                         return new Environment(UsersToolTests.this.settings, confDir.getParent());
                     }
                 };
@@ -140,7 +140,7 @@ public class UsersToolTests extends CommandTestCase {
             protected RolesCommand newRolesCommand() {
                 return new RolesCommand() {
                     @Override
-                    protected Environment createEnv(Terminal terminal, Map<String, String> settings) throws UserException {
+                    protected Environment createEnv(Map<String, String> settings) throws UserException {
                         return new Environment(UsersToolTests.this.settings, confDir.getParent());
                     }
                 };
@@ -150,7 +150,7 @@ public class UsersToolTests extends CommandTestCase {
             protected ListCommand newListCommand() {
                 return new ListCommand() {
                     @Override
-                    protected Environment createEnv(Terminal terminal, Map<String, String> settings) throws UserException {
+                    protected Environment createEnv(Map<String, String> settings) throws UserException {
                         return new Environment(UsersToolTests.this.settings, confDir.getParent());
                     }
                 };

plugin/src/test/java/org/elasticsearch/xpack/security/crypto/tool/SystemKeyToolTests.java

@@ -49,7 +49,7 @@ public class SystemKeyToolTests extends CommandTestCase {
         return new SystemKeyTool() {
 
             @Override
-            protected Environment createEnv(Terminal terminal, Map<String, String> settings) throws UserException {
+            protected Environment createEnv(Map<String, String> settings) throws UserException {
                 Settings.Builder builder = Settings.builder();
                 settings.forEach((k,v) -> builder.put(k, v));
                 return TestEnvironment.newEnvironment(builder.build());

plugin/src/test/resources/org/elasticsearch/transport/handlers

@@ -98,7 +98,6 @@ internal:cluster/nodes/indices/shard/store
 internal:cluster/nodes/indices/shard/store[n]
 internal:cluster/shard/failure
 internal:cluster/shard/started
-internal:cluster/snapshot/update_snapshot
 internal:cluster/snapshot/update_snapshot_status
 internal:discovery/zen/fd/master_ping
 internal:discovery/zen/fd/ping