From 876fc5612af8583fa13da82bbf6716472406efbd Mon Sep 17 00:00:00 2001 From: Chris Earle Date: Wed, 13 Dec 2017 13:46:31 -0500 Subject: [PATCH 1/3] [Monitoring] Use index.auto_expand_replicas: 0-1 (elastic/x-pack-elasticsearch#3282) This changes the default behavior of .monitoring indices to be green on one-node clusters, instead of constantly yellow. Note: This only impacts .monitoring* indices. Watcher indices currently still require a replica. Original commit: elastic/x-pack-elasticsearch@6eb8a48a9f0789206e5e5206280310b5f4c1698f --- plugin/src/main/resources/monitoring-alerts.json | 3 ++- plugin/src/main/resources/monitoring-beats.json | 3 ++- plugin/src/main/resources/monitoring-es.json | 3 ++- plugin/src/main/resources/monitoring-kibana.json | 3 ++- plugin/src/main/resources/monitoring-logstash.json | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/plugin/src/main/resources/monitoring-alerts.json b/plugin/src/main/resources/monitoring-alerts.json index cf4289ff647..01186bca77d 100644 --- a/plugin/src/main/resources/monitoring-alerts.json +++ b/plugin/src/main/resources/monitoring-alerts.json @@ -4,7 +4,8 @@ "settings": { "index": { "number_of_shards": 1, - "number_of_replicas": 1, + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", "format": 6, "codec": "best_compression" } diff --git a/plugin/src/main/resources/monitoring-beats.json b/plugin/src/main/resources/monitoring-beats.json index 38088493de4..6ee5af734d8 100644 --- a/plugin/src/main/resources/monitoring-beats.json +++ b/plugin/src/main/resources/monitoring-beats.json @@ -3,7 +3,8 @@ "version": 7000001, "settings": { "index.number_of_shards": 1, - "index.number_of_replicas": 1, + "index.number_of_replicas": 0, + "index.auto_expand_replicas": "0-1", "index.format": 6, "index.codec": "best_compression" }, diff --git a/plugin/src/main/resources/monitoring-es.json b/plugin/src/main/resources/monitoring-es.json index 926edf535ec..a1726a7a74a 100644 --- a/plugin/src/main/resources/monitoring-es.json +++ b/plugin/src/main/resources/monitoring-es.json @@ -3,7 +3,8 @@ "version": 7000001, "settings": { "index.number_of_shards": 1, - "index.number_of_replicas": 1, + "index.number_of_replicas": 0, + "index.auto_expand_replicas": "0-1", "index.format": 6, "index.codec": "best_compression" }, diff --git a/plugin/src/main/resources/monitoring-kibana.json b/plugin/src/main/resources/monitoring-kibana.json index b7e6dfb2d01..e1f057c7d0d 100644 --- a/plugin/src/main/resources/monitoring-kibana.json +++ b/plugin/src/main/resources/monitoring-kibana.json @@ -3,7 +3,8 @@ "version": 7000001, "settings": { "index.number_of_shards": 1, - "index.number_of_replicas": 1, + "index.number_of_replicas": 0, + "index.auto_expand_replicas": "0-1", "index.format": 6, "index.codec": "best_compression" }, diff --git a/plugin/src/main/resources/monitoring-logstash.json b/plugin/src/main/resources/monitoring-logstash.json index be250d150a7..90ee37cc6e0 100644 --- a/plugin/src/main/resources/monitoring-logstash.json +++ b/plugin/src/main/resources/monitoring-logstash.json @@ -3,7 +3,8 @@ "version": 7000001, "settings": { "index.number_of_shards": 1, - "index.number_of_replicas": 1, + "index.number_of_replicas": 0, + "index.auto_expand_replicas": "0-1", "index.format": 6, "index.codec": "best_compression" }, From d24921ea60b124a333abd641474dc1ea1297155d Mon Sep 17 00:00:00 2001 From: Ioannis Kakavas Date: Thu, 14 Dec 2017 13:45:53 +0200 Subject: [PATCH 2/3] Check for existing x-pack directory when running the `users` CLI tool (elastic/x-pack-elasticsearch#3271) Verify that the configuration directory `$ES_PATH_CONF/x-pack` exists before attempting to run any of the `users` commands, and return a helpful error message to the user if it doesn't. Original commit: elastic/x-pack-elasticsearch@6d663b66546bed3b207901abfd33b36bffee6334 --- .../security/authc/file/tool/UsersTool.java | 62 +++++++++++++++---- .../authc/file/tool/UsersToolTests.java | 55 +++++++++++++++- 2 files changed, 103 insertions(+), 14 deletions(-) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/authc/file/tool/UsersTool.java b/plugin/src/main/java/org/elasticsearch/xpack/security/authc/file/tool/UsersTool.java index d2b2bb5f385..b7b918b8ebd 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/authc/file/tool/UsersTool.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/authc/file/tool/UsersTool.java @@ -7,17 +7,17 @@ package org.elasticsearch.xpack.security.authc.file.tool; import joptsimple.OptionSet; import joptsimple.OptionSpec; -import org.elasticsearch.cli.ExitCodes; -import org.elasticsearch.cli.LoggingAwareMultiCommand; -import org.elasticsearch.cli.MultiCommand; import org.elasticsearch.cli.EnvironmentAwareCommand; +import org.elasticsearch.cli.LoggingAwareMultiCommand; import org.elasticsearch.cli.Terminal; import org.elasticsearch.cli.UserException; +import org.elasticsearch.cli.ExitCodes; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.env.Environment; +import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.XPackSettings; import org.elasticsearch.xpack.security.authc.file.FileUserPasswdStore; import org.elasticsearch.xpack.security.authc.file.FileUserRolesStore; @@ -75,7 +75,7 @@ public class UsersTool extends LoggingAwareMultiCommand { return new ListCommand(); } - static class AddUserCommand extends EnvironmentAwareCommand { + static class AddUserCommand extends XPackConfigurationAwareCommand { private final OptionSpec passwordOption; private final OptionSpec rolesOption; @@ -83,6 +83,7 @@ public class UsersTool extends LoggingAwareMultiCommand { AddUserCommand() { super("Adds a native user"); + this.passwordOption = parser.acceptsAll(Arrays.asList("p", "password"), "The user password") .withRequiredArg(); @@ -104,7 +105,7 @@ public class UsersTool extends LoggingAwareMultiCommand { } @Override - protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception { + protected void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception { String username = parseUsername(arguments.values(options), env.settings()); final boolean allowReserved = XPackSettings.RESERVED_REALM_ENABLED_SETTING.get(env.settings()) == false; @@ -138,7 +139,7 @@ public class UsersTool extends LoggingAwareMultiCommand { } } - static class DeleteUserCommand extends EnvironmentAwareCommand { + static class DeleteUserCommand extends XPackConfigurationAwareCommand { private final OptionSpec arguments; @@ -159,7 +160,8 @@ public class UsersTool extends LoggingAwareMultiCommand { } @Override - protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception { + protected void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception { + String username = parseUsername(arguments.values(options), env.settings()); Path passwordFile = FileUserPasswdStore.resolveFile(env); Path rolesFile = FileUserRolesStore.resolveFile(env); @@ -188,7 +190,7 @@ public class UsersTool extends LoggingAwareMultiCommand { } } - static class PasswordCommand extends EnvironmentAwareCommand { + static class PasswordCommand extends XPackConfigurationAwareCommand { private final OptionSpec passwordOption; private final OptionSpec arguments; @@ -213,7 +215,8 @@ public class UsersTool extends LoggingAwareMultiCommand { } @Override - protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception { + protected void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception { + String username = parseUsername(arguments.values(options), env.settings()); char[] password = parsePassword(terminal, passwordOption.value(options)); @@ -230,7 +233,7 @@ public class UsersTool extends LoggingAwareMultiCommand { } } - static class RolesCommand extends EnvironmentAwareCommand { + static class RolesCommand extends XPackConfigurationAwareCommand { private final OptionSpec addOption; private final OptionSpec removeOption; @@ -256,7 +259,8 @@ public class UsersTool extends LoggingAwareMultiCommand { } @Override - protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception { + protected void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception { + String username = parseUsername(arguments.values(options), env.settings()); String[] addRoles = parseRoles(terminal, env, addOption.value(options)); String[] removeRoles = parseRoles(terminal, env, removeOption.value(options)); @@ -299,7 +303,7 @@ public class UsersTool extends LoggingAwareMultiCommand { } } - static class ListCommand extends EnvironmentAwareCommand { + static class ListCommand extends XPackConfigurationAwareCommand { private final OptionSpec arguments; @@ -315,7 +319,8 @@ public class UsersTool extends LoggingAwareMultiCommand { } @Override - protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception { + protected void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception { + String username = null; if (options.has(arguments)) { username = arguments.value(options); @@ -475,4 +480,35 @@ public class UsersTool extends LoggingAwareMultiCommand { return roles; } + + private abstract static class XPackConfigurationAwareCommand extends EnvironmentAwareCommand { + + XPackConfigurationAwareCommand(final String description) { + super(description); + } + + @Override + protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception { + checkConfigurationDir(env); + executeCommand(terminal, options, env); + } + + /** + * Ensure the X-Pack configuration directory exists as a child of $ES_CONF_DIR or return a helpful error message. + */ + private void checkConfigurationDir(Environment env) throws Exception { + Path configDir = env.configFile().resolve(XPackPlugin.NAME); + if (Files.exists(configDir) == false) { + throw new UserException(ExitCodes.CONFIG, String.format(Locale.ROOT, + "Directory %s does not exist. Please ensure " + + "that %s is the configuration directory for Elasticsearch and create directory %s/x-pack manually", + configDir.toString(), + configDir.getParent().toString(), + configDir.toString())); + } + } + + protected abstract void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception; + + } } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/file/tool/UsersToolTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/file/tool/UsersToolTests.java index 2f3f52337b1..28c29ac7d31 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/file/tool/UsersToolTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/file/tool/UsersToolTests.java @@ -11,7 +11,6 @@ import org.apache.lucene.util.IOUtils; import org.elasticsearch.cli.Command; import org.elasticsearch.cli.CommandTestCase; import org.elasticsearch.cli.ExitCodes; -import org.elasticsearch.cli.Terminal; import org.elasticsearch.cli.UserException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.PathUtilsForTesting; @@ -41,6 +40,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import static org.hamcrest.Matchers.containsString; + public class UsersToolTests extends CommandTestCase { // the mock filesystem we use so permissions/users/groups can be modified @@ -488,4 +489,56 @@ public class UsersToolTests extends CommandTestCase { // output should not contain '*' which indicates unknown role assertFalse(output, output.contains("*")); } + + public void testUserAddNoConfig() throws Exception { + Path homeDir = jimfs.getPath("eshome"); + Path xpackConfDir = homeDir.resolve("config").resolve(XPackPlugin.NAME); + IOUtils.rm(confDir); + pathHomeParameter = "-Epath.home=" + homeDir; + fileTypeParameter = "-Expack.security.authc.realms.file.type=file"; + UserException e = expectThrows(UserException.class, () -> { + execute("useradd", pathHomeParameter, fileTypeParameter, "username", "-p", SecuritySettingsSource.TEST_PASSWORD); + }); + assertEquals(ExitCodes.CONFIG, e.exitCode); + assertThat(e.getMessage(), containsString("is the configuration directory for Elasticsearch and create directory")); + } + + public void testUserListNoConfig() throws Exception { + Path homeDir = jimfs.getPath("eshome"); + Path xpackConfDir = homeDir.resolve("config").resolve(XPackPlugin.NAME); + IOUtils.rm(confDir); + pathHomeParameter = "-Epath.home=" + homeDir; + fileTypeParameter = "-Expack.security.authc.realms.file.type=file"; + UserException e = expectThrows(UserException.class, () -> { + execute("list", pathHomeParameter, fileTypeParameter); + }); + assertEquals(ExitCodes.CONFIG, e.exitCode); + assertThat(e.getMessage(), containsString("is the configuration directory for Elasticsearch and create directory")); + } + + public void testUserDelNoConfig() throws Exception { + Path homeDir = jimfs.getPath("eshome"); + Path xpackConfDir = homeDir.resolve("config").resolve(XPackPlugin.NAME); + IOUtils.rm(confDir); + pathHomeParameter = "-Epath.home=" + homeDir; + fileTypeParameter = "-Expack.security.authc.realms.file.type=file"; + UserException e = expectThrows(UserException.class, () -> { + execute("userdel", pathHomeParameter, fileTypeParameter, "username"); + }); + assertEquals(ExitCodes.CONFIG, e.exitCode); + assertThat(e.getMessage(), containsString("is the configuration directory for Elasticsearch and create directory")); + } + + public void testListUserRolesNoConfig() throws Exception { + Path homeDir = jimfs.getPath("eshome"); + Path xpackConfDir = homeDir.resolve("config").resolve(XPackPlugin.NAME); + IOUtils.rm(confDir); + pathHomeParameter = "-Epath.home=" + homeDir; + fileTypeParameter = "-Expack.security.authc.realms.file.type=file"; + UserException e = expectThrows(UserException.class, () -> { + execute("roles", pathHomeParameter, fileTypeParameter, "username"); + }); + assertEquals(ExitCodes.CONFIG, e.exitCode); + assertThat(e.getMessage(), containsString("is the configuration directory for Elasticsearch and create directory")); + } } From 51b80aa99834fc7178ac6e1c134f2b78077841da Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Thu, 14 Dec 2017 12:17:25 +0000 Subject: [PATCH 3/3] Fixes test to support BytesSizeValue changes (elastic/x-pack-elasticsearch#3321) Original commit: elastic/x-pack-elasticsearch@0bc8b0e847fb0c893ae4f46c0edbf87cd80b85f8 --- .../elasticsearch/xpack/ml/job/config/AnalysisLimitsTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisLimitsTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisLimitsTests.java index 46b837e54ac..afafa1de687 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisLimitsTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisLimitsTests.java @@ -69,7 +69,7 @@ public class AnalysisLimitsTests extends AbstractSerializingTestCase AnalysisLimits.CONFIG_PARSER.apply(parser, null)); - assertThat(e.getRootCause().getMessage(), containsString("model_memory_limit must be at least 1 MiB. Value = -4")); + assertThat(e.getRootCause().getMessage(), containsString("Values less than -1 bytes are not supported: -4mb")); } public void testParseModelMemoryLimitGivenZeroString() throws IOException {