Merge branch 'master' into feature/sql_2

Original commit: elastic/x-pack-elasticsearch@ded26a3834

plugin/src/main/java/org/elasticsearch/xpack/security/authc/file/tool/UsersTool.java

@@ -7,17 +7,17 @@ package org.elasticsearch.xpack.security.authc.file.tool;
 
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
-import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.LoggingAwareMultiCommand;
-import org.elasticsearch.cli.MultiCommand;
 import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.LoggingAwareMultiCommand;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
+import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.xpack.XPackSettings;
 import org.elasticsearch.xpack.security.authc.file.FileUserPasswdStore;
 import org.elasticsearch.xpack.security.authc.file.FileUserRolesStore;
@@ -75,7 +75,7 @@ public class UsersTool extends LoggingAwareMultiCommand {
         return new ListCommand();
     }
 
-    static class AddUserCommand extends EnvironmentAwareCommand {
+    static class AddUserCommand extends XPackConfigurationAwareCommand {
 
         private final OptionSpec<String> passwordOption;
         private final OptionSpec<String> rolesOption;
@@ -83,6 +83,7 @@ public class UsersTool extends LoggingAwareMultiCommand {
 
         AddUserCommand() {
             super("Adds a native user");
+
             this.passwordOption = parser.acceptsAll(Arrays.asList("p", "password"),
                 "The user password")
                 .withRequiredArg();
@@ -104,7 +105,7 @@ public class UsersTool extends LoggingAwareMultiCommand {
         }
 
         @Override
-        protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        protected void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception {
 
             String username = parseUsername(arguments.values(options), env.settings());
             final boolean allowReserved = XPackSettings.RESERVED_REALM_ENABLED_SETTING.get(env.settings()) == false;
@@ -138,7 +139,7 @@ public class UsersTool extends LoggingAwareMultiCommand {
         }
     }
 
-    static class DeleteUserCommand extends EnvironmentAwareCommand {
+    static class DeleteUserCommand extends XPackConfigurationAwareCommand {
 
         private final OptionSpec<String> arguments;
 
@@ -159,7 +160,8 @@ public class UsersTool extends LoggingAwareMultiCommand {
         }
 
         @Override
-        protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        protected void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception {
+
             String username = parseUsername(arguments.values(options), env.settings());
             Path passwordFile = FileUserPasswdStore.resolveFile(env);
             Path rolesFile = FileUserRolesStore.resolveFile(env);
@@ -188,7 +190,7 @@ public class UsersTool extends LoggingAwareMultiCommand {
         }
     }
 
-    static class PasswordCommand extends EnvironmentAwareCommand {
+    static class PasswordCommand extends XPackConfigurationAwareCommand {
 
         private final OptionSpec<String> passwordOption;
         private final OptionSpec<String> arguments;
@@ -213,7 +215,8 @@ public class UsersTool extends LoggingAwareMultiCommand {
         }
 
         @Override
-        protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        protected void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception {
+
             String username = parseUsername(arguments.values(options), env.settings());
             char[] password = parsePassword(terminal, passwordOption.value(options));
 
@@ -230,7 +233,7 @@ public class UsersTool extends LoggingAwareMultiCommand {
         }
     }
 
-    static class RolesCommand extends EnvironmentAwareCommand {
+    static class RolesCommand extends XPackConfigurationAwareCommand {
 
         private final OptionSpec<String> addOption;
         private final OptionSpec<String> removeOption;
@@ -256,7 +259,8 @@ public class UsersTool extends LoggingAwareMultiCommand {
         }
 
         @Override
-        protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        protected void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception {
+
             String username = parseUsername(arguments.values(options), env.settings());
             String[] addRoles = parseRoles(terminal, env, addOption.value(options));
             String[] removeRoles = parseRoles(terminal, env, removeOption.value(options));
@@ -299,7 +303,7 @@ public class UsersTool extends LoggingAwareMultiCommand {
         }
     }
 
-    static class ListCommand extends EnvironmentAwareCommand {
+    static class ListCommand extends XPackConfigurationAwareCommand {
 
         private final OptionSpec<String> arguments;
 
@@ -315,7 +319,8 @@ public class UsersTool extends LoggingAwareMultiCommand {
         }
 
         @Override
-        protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        protected void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception {
+
             String username = null;
             if (options.has(arguments)) {
                 username = arguments.value(options);
@@ -475,4 +480,35 @@ public class UsersTool extends LoggingAwareMultiCommand {
 
         return roles;
     }
+
+    private abstract static class XPackConfigurationAwareCommand extends EnvironmentAwareCommand {
+
+        XPackConfigurationAwareCommand(final String description) {
+            super(description);
+        }
+
+        @Override
+        protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+            checkConfigurationDir(env);
+            executeCommand(terminal, options, env);
+        }
+
+        /**
+         * Ensure the X-Pack configuration directory exists as a child of $ES_CONF_DIR or return a helpful error message.
+         */
+        private void checkConfigurationDir(Environment env) throws Exception {
+            Path configDir = env.configFile().resolve(XPackPlugin.NAME);
+            if (Files.exists(configDir) == false) {
+                throw new UserException(ExitCodes.CONFIG, String.format(Locale.ROOT,
+                        "Directory %s does not exist. Please ensure " +
+                                "that %s is the configuration directory for Elasticsearch and create directory %s/x-pack manually",
+                        configDir.toString(),
+                        configDir.getParent().toString(),
+                        configDir.toString()));
+            }
+        }
+
+        protected abstract void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception;
+
+    }
 }

plugin/src/main/resources/monitoring-alerts.json

@@ -4,7 +4,8 @@
   "settings": {
     "index": {
       "number_of_shards": 1,
-      "number_of_replicas": 1,
+      "number_of_replicas": 0,
+      "auto_expand_replicas": "0-1",
       "format": 6,
       "codec": "best_compression"
     }

plugin/src/main/resources/monitoring-beats.json

@@ -3,7 +3,8 @@
   "version": 7000001,
   "settings": {
     "index.number_of_shards": 1,
-    "index.number_of_replicas": 1,
+    "index.number_of_replicas": 0,
+    "index.auto_expand_replicas": "0-1",
     "index.format": 6,
     "index.codec": "best_compression"
   },

plugin/src/main/resources/monitoring-es.json

@@ -3,7 +3,8 @@
   "version": 7000001,
   "settings": {
     "index.number_of_shards": 1,
-    "index.number_of_replicas": 1,
+    "index.number_of_replicas": 0,
+    "index.auto_expand_replicas": "0-1",
     "index.format": 6,
     "index.codec": "best_compression"
   },

plugin/src/main/resources/monitoring-kibana.json

@@ -3,7 +3,8 @@
   "version": 7000001,
   "settings": {
     "index.number_of_shards": 1,
-    "index.number_of_replicas": 1,
+    "index.number_of_replicas": 0,
+    "index.auto_expand_replicas": "0-1",
     "index.format": 6,
     "index.codec": "best_compression"
   },

plugin/src/main/resources/monitoring-logstash.json

@@ -3,7 +3,8 @@
   "version": 7000001,
   "settings": {
     "index.number_of_shards": 1,
-    "index.number_of_replicas": 1,
+    "index.number_of_replicas": 0,
+    "index.auto_expand_replicas": "0-1",
     "index.format": 6,
     "index.codec": "best_compression"
   },

plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisLimitsTests.java

@@ -69,7 +69,7 @@ public class AnalysisLimitsTests extends AbstractSerializingTestCase<AnalysisLim
         String json = "{\"model_memory_limit\":\"-4MB\"}";
         XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(NamedXContentRegistry.EMPTY, json);
         ParsingException e = expectThrows(ParsingException.class, () -> AnalysisLimits.CONFIG_PARSER.apply(parser, null));
-        assertThat(e.getRootCause().getMessage(), containsString("model_memory_limit must be at least 1 MiB. Value = -4"));
+        assertThat(e.getRootCause().getMessage(), containsString("Values less than -1 bytes are not supported: -4mb"));
     }
 
     public void testParseModelMemoryLimitGivenZeroString() throws IOException {

plugin/src/test/java/org/elasticsearch/xpack/security/authc/file/tool/UsersToolTests.java

@@ -11,7 +11,6 @@ import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.cli.Command;
 import org.elasticsearch.cli.CommandTestCase;
 import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.PathUtilsForTesting;
@@ -41,6 +40,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
+import static org.hamcrest.Matchers.containsString;
+
 public class UsersToolTests extends CommandTestCase {
 
     // the mock filesystem we use so permissions/users/groups can be modified
@@ -488,4 +489,56 @@ public class UsersToolTests extends CommandTestCase {
         // output should not contain '*' which indicates unknown role
         assertFalse(output, output.contains("*"));
     }
+
+    public void testUserAddNoConfig() throws Exception {
+        Path homeDir = jimfs.getPath("eshome");
+        Path xpackConfDir = homeDir.resolve("config").resolve(XPackPlugin.NAME);
+        IOUtils.rm(confDir);
+        pathHomeParameter = "-Epath.home=" + homeDir;
+        fileTypeParameter = "-Expack.security.authc.realms.file.type=file";
+        UserException e = expectThrows(UserException.class, () -> {
+            execute("useradd", pathHomeParameter, fileTypeParameter, "username", "-p", SecuritySettingsSource.TEST_PASSWORD);
+        });
+        assertEquals(ExitCodes.CONFIG, e.exitCode);
+        assertThat(e.getMessage(), containsString("is the configuration directory for Elasticsearch and create directory"));
+    }
+
+    public void testUserListNoConfig() throws Exception {
+        Path homeDir = jimfs.getPath("eshome");
+        Path xpackConfDir = homeDir.resolve("config").resolve(XPackPlugin.NAME);
+        IOUtils.rm(confDir);
+        pathHomeParameter = "-Epath.home=" + homeDir;
+        fileTypeParameter = "-Expack.security.authc.realms.file.type=file";
+        UserException e = expectThrows(UserException.class, () -> {
+            execute("list", pathHomeParameter, fileTypeParameter);
+        });
+        assertEquals(ExitCodes.CONFIG, e.exitCode);
+        assertThat(e.getMessage(), containsString("is the configuration directory for Elasticsearch and create directory"));
+    }
+
+    public void testUserDelNoConfig() throws Exception {
+        Path homeDir = jimfs.getPath("eshome");
+        Path xpackConfDir = homeDir.resolve("config").resolve(XPackPlugin.NAME);
+        IOUtils.rm(confDir);
+        pathHomeParameter = "-Epath.home=" + homeDir;
+        fileTypeParameter = "-Expack.security.authc.realms.file.type=file";
+        UserException e = expectThrows(UserException.class, () -> {
+            execute("userdel", pathHomeParameter, fileTypeParameter, "username");
+        });
+        assertEquals(ExitCodes.CONFIG, e.exitCode);
+        assertThat(e.getMessage(), containsString("is the configuration directory for Elasticsearch and create directory"));
+    }
+
+    public void testListUserRolesNoConfig() throws Exception {
+        Path homeDir = jimfs.getPath("eshome");
+        Path xpackConfDir = homeDir.resolve("config").resolve(XPackPlugin.NAME);
+        IOUtils.rm(confDir);
+        pathHomeParameter = "-Epath.home=" + homeDir;
+        fileTypeParameter = "-Expack.security.authc.realms.file.type=file";
+        UserException e = expectThrows(UserException.class, () -> {
+            execute("roles", pathHomeParameter, fileTypeParameter, "username");
+        });
+        assertEquals(ExitCodes.CONFIG, e.exitCode);
+        assertThat(e.getMessage(), containsString("is the configuration directory for Elasticsearch and create directory"));
+    }
 }