From 447f2246772adc22e95f8619ba593f0fddc5c9d7 Mon Sep 17 00:00:00 2001 From: Aaron Bull Schaefer Date: Wed, 13 Sep 2017 16:01:21 -0700 Subject: [PATCH 1/7] Add releaseTest option to CI script (elastic/x-pack-elasticsearch#2482) This option runs a normal check but with `-Dsnapshot=false` (the flag used to indicate a release build). Related to https://github.com/elastic/infra/issues/2759 and https://github.com/elastic/infra/issues/2739 from the ES side. Original commit: elastic/x-pack-elasticsearch@e674e68905186792c724c0fb925dbae3caf3e9e1 --- dev-tools/ci | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dev-tools/ci b/dev-tools/ci index 803f2c035a6..b501cb30216 100755 --- a/dev-tools/ci +++ b/dev-tools/ci @@ -57,6 +57,16 @@ case $key in "-Dtests.badapples=true" ) ;; + releaseTest) + GRADLE_CLI_ARGS=( + "--info" + "check" + "-Dtests.network=true" + "-Dtests.badapples=true" + "-Dbuild.snapshot=false" + "-Dtests.jvm.argline=-Dbuild.snapshot=false" + ) + ;; jdk9) GRADLE_CLI_ARGS=( "-Pxpack.kibana.build=false" From 89d6c7e01ee273468fd7d32add506c3ce6c8f529 Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Wed, 13 Sep 2017 17:16:06 -0700 Subject: [PATCH 2/7] [DOCS] Create reference for users command (elastic/x-pack-elasticsearch#2480) Original commit: elastic/x-pack-elasticsearch@d0afe8a20defc2b91b20ea28f7e67c9f4b2bb591 --- docs/en/commands/index.asciidoc | 16 +++ docs/en/commands/users-command.asciidoc | 138 ++++++++++++++++++++++++ docs/en/index.asciidoc | 3 + 3 files changed, 157 insertions(+) create mode 100644 docs/en/commands/index.asciidoc create mode 100644 docs/en/commands/users-command.asciidoc diff --git a/docs/en/commands/index.asciidoc b/docs/en/commands/index.asciidoc new file mode 100644 index 00000000000..c1a063bc13d --- /dev/null +++ b/docs/en/commands/index.asciidoc @@ -0,0 +1,16 @@ +[role="xpack"] +[[xpack-commands]] += {xpack} Commands + +[partintro] +-- + +{xpack} includes commands that help you configure security: + +//* <> +//* <> +* <> + +-- + +include::users-command.asciidoc[] diff --git a/docs/en/commands/users-command.asciidoc b/docs/en/commands/users-command.asciidoc new file mode 100644 index 00000000000..6a889e86346 --- /dev/null +++ b/docs/en/commands/users-command.asciidoc @@ -0,0 +1,138 @@ +[role="xpack"] +[[users-command]] +== Users Command +++++ +users +++++ + +If you use file-based user authentication, the `users` command enables you to +add and remove users, assign user roles, and manage passwords. + +[float] +=== Synopsis + +[source,shell] +-------------------------------------------------- +bin/x-pack/users +([useradd ] [-p ] [-r ]) | +([list] ) | +([passwd ] [-p ]) | +([roles ] [-a ] [-r ]) | +([userdel ]) +-------------------------------------------------- + +[float] +=== Description + +If you use the built-in `file` internal realm, users are defined in local files +on each node in the cluster. + +Usernames and roles must be at least 1 and no more than 1024 characters. They +can contain alphanumeric characters (`a-z`, `A-Z`, `0-9`), spaces, punctuation, +and printable symbols in the +https://en.wikipedia.org/wiki/Basic_Latin_(Unicode_block)[Basic Latin (ASCII) block]. +Leading or trailing whitespace is not allowed. + +Passwords must be at least 6 characters long. + +For more information, see {xpack-ref}/file-realm.html[File-based User Authentication]. + +[float] +=== Parameters + +`-a `:: If used with the `roles` parameter, adds a comma-separated list +of roles to a user. + +//`-h, --help`:: Returns all of the command parameters. + +`list`:: List the users that are registered with the `file` realm +on the local node. If you also specify a user name, the command provides +information for that user. + +`-p `:: Specifies the user's password. If you do not specify this +parameter, the command prompts you for the password. ++ +-- +TIP: Omit the `-p` option to keep +plaintext passwords out of the terminal session's command history. + +-- + +`passwd `:: Resets a user's password. You can specify the new +password directly with the `-p` parameter. + +`-r `:: +* If used with the `useradd` parameter, defines a user's roles. This option +accepts a comma-separated list of role names to assign to the user. +* If used with the `roles` parameter, removes a comma-separated list of roles +from a user. + +`roles`:: Manages the roles of a particular user. You can combine adding and +removing roles within the same command to change a user's roles. + +//`-s, --silent`:: Shows minimal output. + +`useradd `:: Adds a user to your local node. + +`userdel `:: Deletes a user from your local node. + +//`-v, --verbose`:: Shows verbose output. + +[float] +=== Authorization + +To ensure that {es} can read the user and role information at startup, run +`users useradd` as the same user you use to run {es}. Running the command as +root or some other user updates the permissions for the `users` and `users_roles` +files and prevents {es} from accessing them. + +[float] +=== Examples + +The following example adds a new user named `jacknich` to the `file` realm. The +password for this user is `theshining`, and this user is associated with the +`network` and `monitoring` roles. + +[source,shell] +------------------------------------------------------------------- +bin/x-pack/users useradd jacknich -p theshining -r network,monitoring +------------------------------------------------------------------- + +The following example lists the users that are registered with the `file` realm +on the local node: + +[source, shell] +---------------------------------- +bin/x-pack/users list +rdeniro : admin +alpacino : power_user +jacknich : monitoring,network +---------------------------------- + +Users are in the left-hand column and their corresponding roles are listed in +the right-hand column. + +The following example resets the `jacknich` user's password: + +[source,shell] +-------------------------------------------------- +bin/x-pack/users passwd jachnich +-------------------------------------------------- + +Since the `-p` parameter was omitted, the command prompts you to enter and +confirm a password in interactive mode. + +The following example removes the `network` and `monitoring` roles from the +`jacknich` user and adds the `user` role: + +[source,shell] +------------------------------------------------------------ +bin/x-pack/users roles jacknich -r network,monitoring -a user +------------------------------------------------------------ + +The following example deletes the `jacknich` user: + +[source,shell] +-------------------------------------------------- +bin/x-pack/users userdel jacknich +-------------------------------------------------- diff --git a/docs/en/index.asciidoc b/docs/en/index.asciidoc index d865b7e2834..674adcce7d8 100644 --- a/docs/en/index.asciidoc +++ b/docs/en/index.asciidoc @@ -20,5 +20,8 @@ include::{es-repo-dir}/reference/index-shared2.asciidoc[] :edit_url!: include::rest-api/index.asciidoc[] +:edit_url!: +include::commands/index.asciidoc[] + :edit_url: include::{es-repo-dir}/reference/index-shared3.asciidoc[] From 9ea36ef7715aa0bb58f0373133ae53eb3d00056a Mon Sep 17 00:00:00 2001 From: lcawley Date: Wed, 13 Sep 2017 17:21:15 -0700 Subject: [PATCH 3/7] [DOCS] Added tip in users command Original commit: elastic/x-pack-elasticsearch@3fb4e1819c3d629ffe7ff9cbaa1993c9d895b5e0 --- docs/en/commands/users-command.asciidoc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/en/commands/users-command.asciidoc b/docs/en/commands/users-command.asciidoc index 6a889e86346..98a3a6acf32 100644 --- a/docs/en/commands/users-command.asciidoc +++ b/docs/en/commands/users-command.asciidoc @@ -37,6 +37,11 @@ Passwords must be at least 6 characters long. For more information, see {xpack-ref}/file-realm.html[File-based User Authentication]. +TIP: To ensure that {es} can read the user and role information at startup, run +`users useradd` as the same user you use to run {es}. Running the command as +root or some other user updates the permissions for the `users` and `users_roles` +files and prevents {es} from accessing them. + [float] === Parameters @@ -78,13 +83,8 @@ removing roles within the same command to change a user's roles. //`-v, --verbose`:: Shows verbose output. -[float] -=== Authorization - -To ensure that {es} can read the user and role information at startup, run -`users useradd` as the same user you use to run {es}. Running the command as -root or some other user updates the permissions for the `users` and `users_roles` -files and prevents {es} from accessing them. +//[float] +//=== Authorization [float] === Examples From 4f3e740ba897fad42d2a0aa87b912c7012a93db4 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 13 Sep 2017 21:30:51 -0400 Subject: [PATCH 4/7] Refactor bootstrap check results and error messages This commit refactors the X-Pack bootstrap checks to respond to a change in core Elasticsearch where the checks now return a single result object. Relates elastic/x-pack-elasticsearch#2495 Original commit: elastic/x-pack-elasticsearch@230b05052975cf91ff468395d604b3ab4d2d870b --- .../security/PkiRealmBootstrapCheck.java | 18 ++++---- .../security/TokenSSLBootstrapCheck.java | 27 +++++++----- .../RoleMappingFileBootstrapCheck.java | 17 +++----- .../xpack/ssl/SSLBootstrapCheck.java | 15 +++---- .../EncryptSensitiveDataBootstrapCheck.java | 41 ++++++++++--------- .../security/PkiRealmBootstrapCheckTests.java | 18 ++++---- .../security/TokenSSLBootsrapCheckTests.java | 12 +++--- .../RoleMappingFileBootstrapCheckTests.java | 31 +++++++------- .../xpack/ssl/SSLBootstrapCheckTests.java | 12 +++--- ...cryptSensitiveDataBootstrapCheckTests.java | 7 ++-- 10 files changed, 99 insertions(+), 99 deletions(-) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/PkiRealmBootstrapCheck.java b/plugin/src/main/java/org/elasticsearch/xpack/security/PkiRealmBootstrapCheck.java index ad31761cd24..9a79333efbe 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/PkiRealmBootstrapCheck.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/PkiRealmBootstrapCheck.java @@ -31,7 +31,7 @@ class PkiRealmBootstrapCheck implements BootstrapCheck { * least one network communication layer. */ @Override - public boolean check(BootstrapContext context) { + public BootstrapCheckResult check(BootstrapContext context) { final Settings settings = context.settings; final boolean pkiRealmEnabled = settings.getGroups(RealmSettings.PREFIX).values().stream() .filter(s -> PkiRealm.TYPE.equals(s.get("type"))) @@ -42,34 +42,30 @@ class PkiRealmBootstrapCheck implements BootstrapCheck { Settings httpSSLSettings = SSLService.getHttpTransportSSLSettings(settings); final boolean httpClientAuth = sslService.isSSLClientAuthEnabled(httpSSLSettings); if (httpSsl && httpClientAuth) { - return false; + return BootstrapCheckResult.success(); } // Default Transport final Settings transportSSLSettings = settings.getByPrefix(setting("transport.ssl.")); final boolean clientAuthEnabled = sslService.isSSLClientAuthEnabled(transportSSLSettings); if (clientAuthEnabled) { - return false; + return BootstrapCheckResult.success(); } // Transport Profiles Map groupedSettings = settings.getGroups("transport.profiles."); for (Map.Entry entry : groupedSettings.entrySet()) { if (sslService.isSSLClientAuthEnabled(SecurityNetty4Transport.profileSslSettings(entry.getValue()), transportSSLSettings)) { - return false; + return BootstrapCheckResult.success(); } } - return true; + return BootstrapCheckResult.failure( + "a PKI realm is enabled but cannot be used as neither HTTP or Transport have SSL and client authentication enabled"); } else { - return false; + return BootstrapCheckResult.success(); } } - @Override - public String errorMessage() { - return "A PKI realm is enabled but cannot be used as neither HTTP or Transport have SSL and client authentication enabled"; - } - @Override public boolean alwaysEnforce() { return true; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/TokenSSLBootstrapCheck.java b/plugin/src/main/java/org/elasticsearch/xpack/security/TokenSSLBootstrapCheck.java index cf3a1b48ae3..dfa3681c3f8 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/TokenSSLBootstrapCheck.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/TokenSSLBootstrapCheck.java @@ -11,24 +11,29 @@ import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.XPackSettings; +import java.util.Locale; + /** * Bootstrap check to ensure that the user has enabled HTTPS when using the token service */ final class TokenSSLBootstrapCheck implements BootstrapCheck { @Override - public boolean check(BootstrapContext context) { - if (NetworkModule.HTTP_ENABLED.get(context.settings)) { - return XPackSettings.HTTP_SSL_ENABLED.get(context.settings) == false && XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.get - (context.settings); + public BootstrapCheckResult check(BootstrapContext context) { + final Boolean httpEnabled = NetworkModule.HTTP_ENABLED.get(context.settings); + final Boolean httpsEnabled = XPackSettings.HTTP_SSL_ENABLED.get(context.settings); + final Boolean tokenServiceEnabled = XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.get(context.settings); + if (httpEnabled && httpsEnabled == false && tokenServiceEnabled) { + final String message = String.format( + Locale.ROOT, + "HTTPS is required in order to use the token service; " + + "please enable HTTPS using the [%s] setting or disable the token service using the [%s] setting", + XPackSettings.HTTP_SSL_ENABLED.getKey(), + XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey()); + return BootstrapCheckResult.failure(message); + } else { + return BootstrapCheckResult.success(); } - return false; } - @Override - public String errorMessage() { - return "HTTPS is required in order to use the token service. Please enable HTTPS using the [" + - XPackSettings.HTTP_SSL_ENABLED.getKey() + "] setting or disable the token service using the [" + - XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey() + "] setting."; - } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/authc/support/RoleMappingFileBootstrapCheck.java b/plugin/src/main/java/org/elasticsearch/xpack/security/authc/support/RoleMappingFileBootstrapCheck.java index 9b6e0880401..b948920ddcc 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/authc/support/RoleMappingFileBootstrapCheck.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/authc/support/RoleMappingFileBootstrapCheck.java @@ -20,30 +20,22 @@ public class RoleMappingFileBootstrapCheck implements BootstrapCheck { private final RealmConfig realmConfig; private final Path path; - private final SetOnce error = new SetOnce<>(); - - public RoleMappingFileBootstrapCheck(RealmConfig config, Path path) { + RoleMappingFileBootstrapCheck(RealmConfig config, Path path) { this.realmConfig = config; this.path = path; } @Override - public boolean check(BootstrapContext context) { + public BootstrapCheckResult check(BootstrapContext context) { try { DnRoleMapper.parseFile(path, realmConfig.logger(getClass()), realmConfig.type(), realmConfig.name(), true); - return false; + return BootstrapCheckResult.success(); } catch (Exception e) { - error.set(e.getMessage()); - return true; + return BootstrapCheckResult.failure(e.getMessage()); } } - @Override - public String errorMessage() { - return error.get(); - } - @Override public boolean alwaysEnforce() { return true; @@ -56,4 +48,5 @@ public class RoleMappingFileBootstrapCheck implements BootstrapCheck { } return null; } + } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ssl/SSLBootstrapCheck.java b/plugin/src/main/java/org/elasticsearch/xpack/ssl/SSLBootstrapCheck.java index 4146215369a..1cae925f8f8 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ssl/SSLBootstrapCheck.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ssl/SSLBootstrapCheck.java @@ -42,10 +42,15 @@ public final class SSLBootstrapCheck implements BootstrapCheck { } @Override - public boolean check(BootstrapContext context) { + public BootstrapCheckResult check(BootstrapContext context) { final Settings transportSSLSettings = context.settings.getByPrefix(XPackSettings.TRANSPORT_SSL_PREFIX); - return sslService.sslConfiguration(transportSSLSettings).keyConfig() == KeyConfig.NONE - || isDefaultCACertificateTrusted() || isDefaultPrivateKeyUsed(); + if (sslService.sslConfiguration(transportSSLSettings).keyConfig() == KeyConfig.NONE + || isDefaultCACertificateTrusted() || isDefaultPrivateKeyUsed()) { + return BootstrapCheckResult.failure( + "default SSL key and certificate do not provide security; please generate keys and certificates"); + } else { + return BootstrapCheckResult.success(); + } } /** @@ -91,8 +96,4 @@ public final class SSLBootstrapCheck implements BootstrapCheck { .anyMatch(defaultPrivateKey::equals); } - @Override - public String errorMessage() { - return "Default SSL key and certificate do not provide security; please generate keys and certificates"; - } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheck.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheck.java index d9eafe414dc..84fc4978c31 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheck.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheck.java @@ -23,27 +23,28 @@ final class EncryptSensitiveDataBootstrapCheck implements BootstrapCheck { } @Override - public boolean check(BootstrapContext context) { - return Watcher.ENCRYPT_SENSITIVE_DATA_SETTING.get(context.settings) - && Watcher.ENCRYPTION_KEY_SETTING.exists(context.settings) == false; - } - - @Override - public String errorMessage() { - final Path sysKeyPath = environment.configFile().resolve(XPackPlugin.NAME).resolve("system_key").toAbsolutePath(); - if (Files.exists(sysKeyPath)) { - return "Encryption of sensitive data requires the key to be placed in the secure setting store. Run " + - "'bin/elasticsearch-keystore add-file " + Watcher.ENCRYPTION_KEY_SETTING.getKey() + " " + - environment.configFile().resolve(XPackPlugin.NAME).resolve("system_key").toAbsolutePath() + - "' to import the file.\nAfter importing, the system_key file should be removed from the " + - "filesystem.\nRepeat this on every node in the cluster."; + public BootstrapCheckResult check(BootstrapContext context) { + if (Watcher.ENCRYPT_SENSITIVE_DATA_SETTING.get(context.settings) + && Watcher.ENCRYPTION_KEY_SETTING.exists(context.settings) == false) { + final Path systemKeyPath = environment.configFile().resolve(XPackPlugin.NAME).resolve("system_key").toAbsolutePath(); + final String message; + if (Files.exists(systemKeyPath)) { + message = "Encryption of sensitive data requires the key to be placed in the secure setting store. Run " + + "'bin/elasticsearch-keystore add-file " + Watcher.ENCRYPTION_KEY_SETTING.getKey() + " " + + systemKeyPath + + "' to import the file.\nAfter importing, the system_key file should be removed from the " + + "filesystem.\nRepeat this on every node in the cluster."; + } else { + message = "Encryption of sensitive data requires a key to be placed in the secure setting store. First run the " + + "bin/x-pack/syskeygen tool to generate a key file.\nThen run 'bin/elasticsearch-keystore add-file " + + Watcher.ENCRYPTION_KEY_SETTING.getKey() + " " + + systemKeyPath + "' to import the key into" + + " the secure setting store. Finally, remove the system_key file from the filesystem.\n" + + "Repeat this on every node in the cluster"; + } + return BootstrapCheckResult.failure(message); } else { - return "Encryption of sensitive data requires a key to be placed in the secure setting store. First run the " + - "bin/x-pack/syskeygen tool to generate a key file.\nThen run 'bin/elasticsearch-keystore add-file " + - Watcher.ENCRYPTION_KEY_SETTING.getKey() + " " + - environment.configFile().resolve(XPackPlugin.NAME).resolve("system_key").toAbsolutePath() + "' to import the key into" + - " the secure setting store. Finally, remove the system_key file from the filesystem.\n" + - "Repeat this on every node in the cluster"; + return BootstrapCheckResult.success(); } } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/PkiRealmBootstrapCheckTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/PkiRealmBootstrapCheckTests.java index 0ce3ebd9b2a..e5c433341c5 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/PkiRealmBootstrapCheckTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/PkiRealmBootstrapCheckTests.java @@ -17,7 +17,7 @@ public class PkiRealmBootstrapCheckTests extends ESTestCase { public void testPkiRealmBootstrapDefault() throws Exception { assertFalse(new PkiRealmBootstrapCheck(new SSLService(Settings.EMPTY, new Environment(Settings.builder().put("path.home", createTempDir()).build()))).check((new BootstrapContext(Settings - .EMPTY, null)))); + .EMPTY, null))).isFailure()); } public void testBootstrapCheckWithPkiRealm() throws Exception { @@ -26,42 +26,42 @@ public class PkiRealmBootstrapCheckTests extends ESTestCase { .put("path.home", createTempDir()) .build(); Environment env = new Environment(settings); - assertFalse(new PkiRealmBootstrapCheck(new SSLService(settings, env)).check(new BootstrapContext(settings, null))); + assertFalse(new PkiRealmBootstrapCheck(new SSLService(settings, env)).check(new BootstrapContext(settings, null)).isFailure()); // disable client auth default settings = Settings.builder().put(settings) .put("xpack.ssl.client_authentication", "none") .build(); env = new Environment(settings); - assertTrue(new PkiRealmBootstrapCheck(new SSLService(settings, env)).check(new BootstrapContext(settings, null))); + assertTrue(new PkiRealmBootstrapCheck(new SSLService(settings, env)).check(new BootstrapContext(settings, null)).isFailure()); // enable ssl for http settings = Settings.builder().put(settings) .put("xpack.security.http.ssl.enabled", true) .build(); env = new Environment(settings); - assertTrue(new PkiRealmBootstrapCheck(new SSLService(settings, env)).check(new BootstrapContext(settings, null))); + assertTrue(new PkiRealmBootstrapCheck(new SSLService(settings, env)).check(new BootstrapContext(settings, null)).isFailure()); // enable client auth for http settings = Settings.builder().put(settings) .put("xpack.security.http.ssl.client_authentication", randomFrom("required", "optional")) .build(); env = new Environment(settings); - assertFalse(new PkiRealmBootstrapCheck(new SSLService(settings, env)).check(new BootstrapContext(settings, null))); + assertFalse(new PkiRealmBootstrapCheck(new SSLService(settings, env)).check(new BootstrapContext(settings, null)).isFailure()); // disable http ssl settings = Settings.builder().put(settings) .put("xpack.security.http.ssl.enabled", false) .build(); env = new Environment(settings); - assertTrue(new PkiRealmBootstrapCheck(new SSLService(settings, env)).check(new BootstrapContext(settings, null))); + assertTrue(new PkiRealmBootstrapCheck(new SSLService(settings, env)).check(new BootstrapContext(settings, null)).isFailure()); // set transport client auth settings = Settings.builder().put(settings) .put("xpack.security.transport.client_authentication", randomFrom("required", "optional")) .build(); env = new Environment(settings); - assertTrue(new PkiRealmBootstrapCheck(new SSLService(settings, env)).check(new BootstrapContext(settings, null))); + assertTrue(new PkiRealmBootstrapCheck(new SSLService(settings, env)).check(new BootstrapContext(settings, null)).isFailure()); // test with transport profile settings = Settings.builder().put(settings) @@ -69,7 +69,7 @@ public class PkiRealmBootstrapCheckTests extends ESTestCase { .put("transport.profiles.foo.xpack.security.ssl.client_authentication", randomFrom("required", "optional")) .build(); env = new Environment(settings); - assertFalse(new PkiRealmBootstrapCheck(new SSLService(settings, env)).check(new BootstrapContext(settings, null))); + assertFalse(new PkiRealmBootstrapCheck(new SSLService(settings, env)).check(new BootstrapContext(settings, null)).isFailure()); } public void testBootstrapCheckWithDisabledRealm() throws Exception { @@ -80,6 +80,6 @@ public class PkiRealmBootstrapCheckTests extends ESTestCase { .put("path.home", createTempDir()) .build(); Environment env = new Environment(settings); - assertFalse(new PkiRealmBootstrapCheck(new SSLService(settings, env)).check(new BootstrapContext(settings, null))); + assertFalse(new PkiRealmBootstrapCheck(new SSLService(settings, env)).check(new BootstrapContext(settings, null)).isFailure()); } } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/TokenSSLBootsrapCheckTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/TokenSSLBootsrapCheckTests.java index 39b33eab0e9..a30cb834784 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/TokenSSLBootsrapCheckTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/TokenSSLBootsrapCheckTests.java @@ -16,29 +16,29 @@ public class TokenSSLBootsrapCheckTests extends ESTestCase { public void testTokenSSLBootstrapCheck() { Settings settings = Settings.EMPTY; - assertFalse(new TokenSSLBootstrapCheck().check(new BootstrapContext(settings, null))); + assertFalse(new TokenSSLBootstrapCheck().check(new BootstrapContext(settings, null)).isFailure()); settings = Settings.builder() .put(NetworkModule.HTTP_ENABLED.getKey(), false) .put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), true).build(); - assertFalse(new TokenSSLBootstrapCheck().check(new BootstrapContext(settings, null))); + assertFalse(new TokenSSLBootstrapCheck().check(new BootstrapContext(settings, null)).isFailure()); settings = Settings.builder().put(XPackSettings.HTTP_SSL_ENABLED.getKey(), true).build(); - assertFalse(new TokenSSLBootstrapCheck().check(new BootstrapContext(settings, null))); + assertFalse(new TokenSSLBootstrapCheck().check(new BootstrapContext(settings, null)).isFailure()); // XPackSettings.HTTP_SSL_ENABLED default false settings = Settings.builder().put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), true).build(); - assertTrue(new TokenSSLBootstrapCheck().check(new BootstrapContext(settings, null))); + assertTrue(new TokenSSLBootstrapCheck().check(new BootstrapContext(settings, null)).isFailure()); settings = Settings.builder() .put(XPackSettings.HTTP_SSL_ENABLED.getKey(), false) .put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), true).build(); - assertTrue(new TokenSSLBootstrapCheck().check(new BootstrapContext(settings, null))); + assertTrue(new TokenSSLBootstrapCheck().check(new BootstrapContext(settings, null)).isFailure()); settings = Settings.builder() .put(XPackSettings.HTTP_SSL_ENABLED.getKey(), false) .put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), true) .put(NetworkModule.HTTP_ENABLED.getKey(), false).build(); - assertFalse(new TokenSSLBootstrapCheck().check(new BootstrapContext(settings, null))); + assertFalse(new TokenSSLBootstrapCheck().check(new BootstrapContext(settings, null)).isFailure()); } } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/support/RoleMappingFileBootstrapCheckTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/support/RoleMappingFileBootstrapCheckTests.java index 629df88dc6c..cbbc0366fb3 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/support/RoleMappingFileBootstrapCheckTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/support/RoleMappingFileBootstrapCheckTests.java @@ -46,7 +46,7 @@ public class RoleMappingFileBootstrapCheckTests extends ESTestCase { final BootstrapCheck check = RoleMappingFileBootstrapCheck.create(config); assertThat(check, notNullValue()); assertThat(check.alwaysEnforce(), equalTo(true)); - assertThat(check.check(new BootstrapContext(settings, null)), equalTo(false)); + assertFalse(check.check(new BootstrapContext(settings, null)).isFailure()); } public void testBootstrapCheckOfMissingFile() { @@ -59,10 +59,11 @@ public class RoleMappingFileBootstrapCheckTests extends ESTestCase { final BootstrapCheck check = RoleMappingFileBootstrapCheck.create(config); assertThat(check, notNullValue()); assertThat(check.alwaysEnforce(), equalTo(true)); - assertThat(check.check(new BootstrapContext(settings, null)), equalTo(true)); - assertThat(check.errorMessage(), containsString("the-realm-name")); - assertThat(check.errorMessage(), containsString(fileName)); - assertThat(check.errorMessage(), containsString("does not exist")); + final BootstrapCheck.BootstrapCheckResult result = check.check(new BootstrapContext(settings, null)); + assertTrue(result.isFailure()); + assertThat(result.getMessage(), containsString("the-realm-name")); + assertThat(result.getMessage(), containsString(fileName)); + assertThat(result.getMessage(), containsString("does not exist")); } public void testBootstrapCheckWithInvalidYaml() throws IOException { @@ -77,10 +78,11 @@ public class RoleMappingFileBootstrapCheckTests extends ESTestCase { final BootstrapCheck check = RoleMappingFileBootstrapCheck.create(config); assertThat(check, notNullValue()); assertThat(check.alwaysEnforce(), equalTo(true)); - assertThat(check.check(new BootstrapContext(settings, null)), equalTo(true)); - assertThat(check.errorMessage(), containsString("the-realm-name")); - assertThat(check.errorMessage(), containsString(file.toString())); - assertThat(check.errorMessage(), containsString("could not read")); + final BootstrapCheck.BootstrapCheckResult result = check.check(new BootstrapContext(settings, null)); + assertTrue(result.isFailure()); + assertThat(result.getMessage(), containsString("the-realm-name")); + assertThat(result.getMessage(), containsString(file.toString())); + assertThat(result.getMessage(), containsString("could not read")); } public void testBootstrapCheckWithInvalidDn() throws IOException { @@ -95,10 +97,11 @@ public class RoleMappingFileBootstrapCheckTests extends ESTestCase { final BootstrapCheck check = RoleMappingFileBootstrapCheck.create(config); assertThat(check, notNullValue()); assertThat(check.alwaysEnforce(), equalTo(true)); - assertThat(check.check(new BootstrapContext(settings, null)), equalTo(true)); - assertThat(check.errorMessage(), containsString("the-realm-name")); - assertThat(check.errorMessage(), containsString(file.toString())); - assertThat(check.errorMessage(), containsString("invalid DN")); - assertThat(check.errorMessage(), containsString("not-a-dn")); + final BootstrapCheck.BootstrapCheckResult result = check.check(new BootstrapContext(settings, null)); + assertTrue(result.isFailure()); + assertThat(result.getMessage(), containsString("the-realm-name")); + assertThat(result.getMessage(), containsString(file.toString())); + assertThat(result.getMessage(), containsString("invalid DN")); + assertThat(result.getMessage(), containsString("not-a-dn")); } } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ssl/SSLBootstrapCheckTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ssl/SSLBootstrapCheckTests.java index 734814cc6b6..22e9e830ae0 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ssl/SSLBootstrapCheckTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ssl/SSLBootstrapCheckTests.java @@ -16,7 +16,7 @@ public class SSLBootstrapCheckTests extends ESTestCase { public void testSSLBootstrapCheckWithNoKey() throws Exception { SSLService sslService = new SSLService(Settings.EMPTY, null); SSLBootstrapCheck bootstrapCheck = new SSLBootstrapCheck(sslService, null); - assertTrue(bootstrapCheck.check(new BootstrapContext(Settings.EMPTY, null))); + assertTrue(bootstrapCheck.check(new BootstrapContext(Settings.EMPTY, null)).isFailure()); } public void testSSLBootstrapCheckWithKey() throws Exception { @@ -33,7 +33,7 @@ public class SSLBootstrapCheckTests extends ESTestCase { .build(); final Environment env = randomBoolean() ? new Environment(settings) : null; SSLBootstrapCheck bootstrapCheck = new SSLBootstrapCheck(new SSLService(settings, env), env); - assertFalse(bootstrapCheck.check(new BootstrapContext(settings, null))); + assertFalse(bootstrapCheck.check(new BootstrapContext(settings, null)).isFailure()); } public void testSSLBootstrapCheckWithDefaultCABeingTrusted() throws Exception { @@ -53,14 +53,14 @@ public class SSLBootstrapCheckTests extends ESTestCase { .build(); final Environment env = randomBoolean() ? new Environment(settings) : null; SSLBootstrapCheck bootstrapCheck = new SSLBootstrapCheck(new SSLService(settings, env), env); - assertTrue(bootstrapCheck.check(new BootstrapContext(settings, null))); + assertTrue(bootstrapCheck.check(new BootstrapContext(settings, null)).isFailure()); settings = Settings.builder().put(settings.filter((s) -> s.contains(".certificate_authorities"))) .put("xpack.security.http.ssl.certificate_authorities", getDataPath("/org/elasticsearch/xpack/ssl/ca.pem").toString()) .build(); bootstrapCheck = new SSLBootstrapCheck(new SSLService(settings, env), env); - assertTrue(bootstrapCheck.check(new BootstrapContext(settings, null))); + assertTrue(bootstrapCheck.check(new BootstrapContext(settings, null)).isFailure()); } public void testSSLBootstrapCheckWithDefaultKeyBeingUsed() throws Exception { @@ -79,7 +79,7 @@ public class SSLBootstrapCheckTests extends ESTestCase { .build(); final Environment env = randomBoolean() ? new Environment(settings) : null; SSLBootstrapCheck bootstrapCheck = new SSLBootstrapCheck(new SSLService(settings, env), env); - assertTrue(bootstrapCheck.check(new BootstrapContext(settings, null))); + assertTrue(bootstrapCheck.check(new BootstrapContext(settings, null)).isFailure()); settings = Settings.builder().put(settings.filter((s) -> s.contains(".http.ssl."))) .put("xpack.security.transport.profiles.foo.xpack.security.ssl.key", @@ -88,6 +88,6 @@ public class SSLBootstrapCheckTests extends ESTestCase { getDataPath("/org/elasticsearch/xpack/ssl/ca.pem").toString()) .build(); bootstrapCheck = new SSLBootstrapCheck(new SSLService(settings, env), env); - assertTrue(bootstrapCheck.check(new BootstrapContext(settings, null))); + assertTrue(bootstrapCheck.check(new BootstrapContext(settings, null)).isFailure()); } } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheckTests.java b/plugin/src/test/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheckTests.java index d2ebab59d7f..81ee272e3c8 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheckTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/watcher/EncryptSensitiveDataBootstrapCheckTests.java @@ -18,7 +18,7 @@ public class EncryptSensitiveDataBootstrapCheckTests extends ESTestCase { Settings settings = Settings.builder().put("path.home", createTempDir()).build(); Environment env = new Environment(settings); EncryptSensitiveDataBootstrapCheck check = new EncryptSensitiveDataBootstrapCheck(env); - assertFalse(check.check(new BootstrapContext(settings, null))); + assertFalse(check.check(new BootstrapContext(settings, null)).isFailure()); assertTrue(check.alwaysEnforce()); } @@ -29,7 +29,7 @@ public class EncryptSensitiveDataBootstrapCheckTests extends ESTestCase { .build(); Environment env = new Environment(settings); EncryptSensitiveDataBootstrapCheck check = new EncryptSensitiveDataBootstrapCheck(env); - assertTrue(check.check(new BootstrapContext(settings, null))); + assertTrue(check.check(new BootstrapContext(settings, null)).isFailure()); } public void testKeyInKeystore() { @@ -42,6 +42,7 @@ public class EncryptSensitiveDataBootstrapCheckTests extends ESTestCase { .build(); Environment env = new Environment(settings); EncryptSensitiveDataBootstrapCheck check = new EncryptSensitiveDataBootstrapCheck(env); - assertFalse(check.check(new BootstrapContext(settings, null))); + assertFalse(check.check(new BootstrapContext(settings, null)).isFailure()); } + } From 7d19264363583ed66b7bc1e1371faac015a9a8b3 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Thu, 14 Sep 2017 12:31:20 +0200 Subject: [PATCH 5/7] [ML-FC] Branch landing feature/ml (elastic/x-pack-elasticsearch#2500) integrate forecasting feature branch into master - add endpoint xpack/ml/job/forecast to request forecasting on data of ml-jobs - current parameters: end time - persists forecast results into shared or own index - different runs are separated by a 'forecast id' relates elastic/x-pack-elasticsearch#1838 Original commit: elastic/x-pack-elasticsearch@f9d701a6bc0ae56775e96cd39a2cdf2f8c48ccd5 --- .../xpack/ml/MachineLearning.java | 8 +- .../xpack/ml/action/ForecastJobAction.java | 234 +++++++++++++ .../persistence/ElasticsearchMappings.java | 20 ++ .../job/persistence/JobResultsPersister.java | 7 + .../autodetect/AutodetectCommunicator.java | 8 + .../process/autodetect/AutodetectProcess.java | 9 + .../autodetect/AutodetectProcessManager.java | 29 ++ .../BlackHoleAutodetectProcess.java | 9 +- .../autodetect/NativeAutodetectProcess.java | 11 +- .../output/AutoDetectResultProcessor.java | 16 + .../autodetect/params/ForecastParams.java | 102 ++++++ .../writer/ControlMsgToProcessWriter.java | 31 +- .../ml/job/results/AutodetectResult.java | 50 ++- .../xpack/ml/job/results/Forecast.java | 308 ++++++++++++++++++ .../xpack/ml/job/results/ForecastStats.java | 114 +++++++ .../ml/job/results/ReservedFieldNames.java | 4 + .../ml/rest/job/RestForecastJobAction.java | 50 +++ .../action/ForecastJobActionRequestTests.java | 35 ++ .../ForecastJobActionResponseTests.java | 23 ++ .../AutodetectResultProcessorIT.java | 18 +- .../params/ForecastParamsTests.java | 44 +++ .../ml/job/results/AutodetectResultTests.java | 15 +- .../ml/job/results/ForecastStatsTests.java | 46 +++ .../xpack/ml/job/results/ForecastTests.java | 68 ++++ .../org/elasticsearch/transport/actions | 1 + .../rest-api-spec/api/xpack.ml.forecast.json | 24 ++ .../rest-api-spec/test/ml/forecast.yml | 41 +++ 27 files changed, 1302 insertions(+), 23 deletions(-) create mode 100644 plugin/src/main/java/org/elasticsearch/xpack/ml/action/ForecastJobAction.java create mode 100644 plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/ForecastParams.java create mode 100644 plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/Forecast.java create mode 100644 plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ForecastStats.java create mode 100644 plugin/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestForecastJobAction.java create mode 100644 plugin/src/test/java/org/elasticsearch/xpack/ml/action/ForecastJobActionRequestTests.java create mode 100644 plugin/src/test/java/org/elasticsearch/xpack/ml/action/ForecastJobActionResponseTests.java create mode 100644 plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/ForecastParamsTests.java create mode 100644 plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastStatsTests.java create mode 100644 plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastTests.java create mode 100644 plugin/src/test/resources/rest-api-spec/api/xpack.ml.forecast.json create mode 100644 plugin/src/test/resources/rest-api-spec/test/ml/forecast.yml diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 68701a5a5d1..57bde0eb70c 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -44,6 +44,7 @@ import org.elasticsearch.xpack.ml.action.DeleteJobAction; import org.elasticsearch.xpack.ml.action.DeleteModelSnapshotAction; import org.elasticsearch.xpack.ml.action.FinalizeJobExecutionAction; import org.elasticsearch.xpack.ml.action.FlushJobAction; +import org.elasticsearch.xpack.ml.action.ForecastJobAction; import org.elasticsearch.xpack.ml.action.GetBucketsAction; import org.elasticsearch.xpack.ml.action.GetCategoriesAction; import org.elasticsearch.xpack.ml.action.GetDatafeedsAction; @@ -108,6 +109,7 @@ import org.elasticsearch.xpack.ml.rest.filter.RestPutFilterAction; import org.elasticsearch.xpack.ml.rest.job.RestCloseJobAction; import org.elasticsearch.xpack.ml.rest.job.RestDeleteJobAction; import org.elasticsearch.xpack.ml.rest.job.RestFlushJobAction; +import org.elasticsearch.xpack.ml.rest.job.RestForecastJobAction; import org.elasticsearch.xpack.ml.rest.job.RestGetJobStatsAction; import org.elasticsearch.xpack.ml.rest.job.RestGetJobsAction; import org.elasticsearch.xpack.ml.rest.job.RestOpenJobAction; @@ -383,7 +385,8 @@ public class MachineLearning implements ActionPlugin { new RestStartDatafeedAction(settings, restController), new RestStopDatafeedAction(settings, restController), new RestDeleteModelSnapshotAction(settings, restController), - new RestDeleteExpiredDataAction(settings, restController) + new RestDeleteExpiredDataAction(settings, restController), + new RestForecastJobAction(settings, restController) ); } @@ -431,7 +434,8 @@ public class MachineLearning implements ActionPlugin { new ActionHandler<>(CompletionPersistentTaskAction.INSTANCE, CompletionPersistentTaskAction.TransportAction.class), new ActionHandler<>(RemovePersistentTaskAction.INSTANCE, RemovePersistentTaskAction.TransportAction.class), new ActionHandler<>(UpdateProcessAction.INSTANCE, UpdateProcessAction.TransportAction.class), - new ActionHandler<>(DeleteExpiredDataAction.INSTANCE, DeleteExpiredDataAction.TransportAction.class) + new ActionHandler<>(DeleteExpiredDataAction.INSTANCE, DeleteExpiredDataAction.TransportAction.class), + new ActionHandler<>(ForecastJobAction.INSTANCE, ForecastJobAction.TransportAction.class) ); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/ForecastJobAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/ForecastJobAction.java new file mode 100644 index 00000000000..6d6673b783d --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/ForecastJobAction.java @@ -0,0 +1,234 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.tasks.BaseTasksResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.ForecastParams; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.xpack.ml.action.ForecastJobAction.Request.END_TIME; + +public class ForecastJobAction extends Action { + + public static final ForecastJobAction INSTANCE = new ForecastJobAction(); + public static final String NAME = "cluster:admin/xpack/ml/job/forecast"; + + private ForecastJobAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends TransportJobTaskAction.JobTaskRequest implements ToXContentObject { + + public static final ParseField END_TIME = new ParseField("end"); + + private static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); + PARSER.declareString(Request::setEndTime, END_TIME); + } + + public static Request parseRequest(String jobId, XContentParser parser) { + Request request = PARSER.apply(parser, null); + if (jobId != null) { + request.jobId = jobId; + } + return request; + } + + private String endTime; + + Request() { + } + + public Request(String jobId) { + super(jobId); + } + + public String getEndTime() { + return endTime; + } + + public void setEndTime(String endTime) { + this.endTime = endTime; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + this.endTime = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(endTime); + } + + @Override + public int hashCode() { + return Objects.hash(jobId, endTime); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && Objects.equals(endTime, other.endTime); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + if (endTime != null) { + builder.field(END_TIME.getPreferredName(), endTime); + } + builder.endObject(); + return builder; + } + } + + static class RequestBuilder extends ActionRequestBuilder { + + RequestBuilder(ElasticsearchClient client, ForecastJobAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends BaseTasksResponse implements Writeable, ToXContentObject { + + private boolean acknowledged; + private long id; + + Response() { + super(null, null); + } + + Response(boolean acknowledged, long id) { + super(null, null); + this.acknowledged = acknowledged; + this.id = id; + } + + public boolean isacknowledged() { + return acknowledged; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + acknowledged = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeBoolean(acknowledged); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("acknowledged", acknowledged); + builder.field("id", id); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + Response response = (Response) o; + return acknowledged == response.acknowledged; + } + + @Override + public int hashCode() { + return Objects.hash(acknowledged); + } + } + + public static class TransportAction extends TransportJobTaskAction { + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, ClusterService clusterService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + AutodetectProcessManager processManager) { + super(settings, ForecastJobAction.NAME, threadPool, clusterService, transportService, actionFilters, + indexNameExpressionResolver, ForecastJobAction.Request::new, ForecastJobAction.Response::new, ThreadPool.Names.SAME, + processManager); + // ThreadPool.Names.SAME, because operations is executed by + // autodetect worker thread + } + + @Override + protected ForecastJobAction.Response readTaskResponse(StreamInput in) throws IOException { + Response response = new Response(); + response.readFrom(in); + return response; + } + + @Override + protected void taskOperation(Request request, OpenJobAction.JobTask task, ActionListener listener) { + ForecastParams.Builder paramsBuilder = ForecastParams.builder(); + if (request.getEndTime() != null) { + paramsBuilder.endTime(request.getEndTime(), END_TIME); + } + + ForecastParams params = paramsBuilder.build(); + processManager.forecastJob(task, params, e -> { + if (e == null) { + listener.onResponse(new Response(true, params.getId())); + } else { + listener.onFailure(e); + } + }); + } + } +} + diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java index 517b7f77393..d538c24f1ec 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java @@ -17,6 +17,7 @@ import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; import org.elasticsearch.xpack.ml.job.results.Bucket; import org.elasticsearch.xpack.ml.job.results.BucketInfluencer; import org.elasticsearch.xpack.ml.job.results.CategoryDefinition; +import org.elasticsearch.xpack.ml.job.results.Forecast; import org.elasticsearch.xpack.ml.job.results.Influence; import org.elasticsearch.xpack.ml.job.results.Influencer; import org.elasticsearch.xpack.ml.job.results.ModelPlot; @@ -289,6 +290,7 @@ public class ElasticsearchMappings { .field(TYPE, DOUBLE) .endObject(); + addForecastFieldsToMapping(builder); addAnomalyRecordFieldsToMapping(builder); addInfluencerFieldsToMapping(builder); addModelSizeStatsFieldsToMapping(builder); @@ -320,6 +322,24 @@ public class ElasticsearchMappings { } } + private static void addForecastFieldsToMapping(XContentBuilder builder) throws IOException { + + // Forecast Output + builder.startObject(Forecast.FORECAST_LOWER.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(Forecast.FORECAST_UPPER.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(Forecast.FORECAST_PREDICTION.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(Forecast.FORECAST_ID.getPreferredName()) + .field(TYPE, LONG) + .endObject(); + } + + /** * AnomalyRecord fields to be added under the 'properties' section of the mapping * @param builder Add properties to this builder diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java index e059c2debf4..06c14dc8674 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java @@ -29,6 +29,7 @@ import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; import org.elasticsearch.xpack.ml.job.results.Bucket; import org.elasticsearch.xpack.ml.job.results.BucketInfluencer; import org.elasticsearch.xpack.ml.job.results.CategoryDefinition; +import org.elasticsearch.xpack.ml.job.results.Forecast; import org.elasticsearch.xpack.ml.job.results.Influencer; import org.elasticsearch.xpack.ml.job.results.ModelPlot; @@ -151,6 +152,12 @@ public class JobResultsPersister extends AbstractComponent { return this; } + public Builder persistForecast(Forecast forecast) { + logger.trace("[{}] ES BULK ACTION: index forecast to index [{}] with ID [{}]", jobId, indexName, forecast.getId()); + indexResult(forecast.getId(), forecast, Forecast.RESULT_TYPE_VALUE); + return this; + } + private void indexResult(String id, ToXContent resultDoc, String resultType) { try (XContentBuilder content = toXContentBuilder(resultDoc)) { bulkRequest.add(new IndexRequest(indexName, DOC_TYPE, id).source(content)); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java index 5875b5249a2..0dcc25c8825 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java @@ -27,6 +27,7 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutoDetectResult import org.elasticsearch.xpack.ml.job.process.autodetect.output.FlushAcknowledgement; import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams; import org.elasticsearch.xpack.ml.job.process.autodetect.params.FlushJobParams; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.ForecastParams; import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; @@ -205,6 +206,13 @@ public class AutodetectCommunicator implements Closeable { }, handler); } + public void forecastJob(ForecastParams params, BiConsumer handler) { + submitOperation(() -> { + autodetectProcess.forecastJob(params); + return null; + }, handler); + } + @Nullable FlushAcknowledgement waitFlushToCompletion(String flushId) { LOGGER.debug("[{}] waiting for flush", job.getId()); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcess.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcess.java index c8ab913c0b8..7d4078a035f 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcess.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcess.java @@ -10,6 +10,7 @@ import org.elasticsearch.xpack.ml.job.config.ModelPlotConfig; import org.elasticsearch.xpack.ml.job.persistence.StateStreamer; import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams; import org.elasticsearch.xpack.ml.job.process.autodetect.params.FlushJobParams; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.ForecastParams; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.ml.job.results.AutodetectResult; @@ -85,6 +86,14 @@ public interface AutodetectProcess extends Closeable { */ String flushJob(FlushJobParams params) throws IOException; + /** + * Do a forecast on a running job. + * + * @param params The forecast parameters + * @throws IOException If the write fails + */ + void forecastJob(ForecastParams params) throws IOException; + /** * Flush the output data stream */ diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java index 1a82777f214..996d0a76586 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java @@ -42,6 +42,7 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.output.FlushAcknowledge import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams; import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams; import org.elasticsearch.xpack.ml.job.process.autodetect.params.FlushJobParams; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.ForecastParams; import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; @@ -49,6 +50,7 @@ import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerFactory; import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer; import org.elasticsearch.xpack.ml.job.process.normalizer.ScoresUpdater; import org.elasticsearch.xpack.ml.job.process.normalizer.ShortCircuitingRenormalizer; +import org.elasticsearch.xpack.ml.job.results.Forecast; import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData.PersistentTask; @@ -240,6 +242,33 @@ public class AutodetectProcessManager extends AbstractComponent { }); } + /** + * Do a forecast for the running job. + * + * @param jobTask The job task + * @param params Forecast parameters + */ + public void forecastJob(JobTask jobTask, ForecastParams params, Consumer handler) { + logger.debug("Forecasting job {}", jobTask.getJobId()); + AutodetectCommunicator communicator = autoDetectCommunicatorByOpenJob.get(jobTask.getAllocationId()); + if (communicator == null) { + String message = String.format(Locale.ROOT, "Cannot forecast because job [%s] is not open", jobTask.getJobId()); + logger.debug(message); + handler.accept(ExceptionsHelper.conflictStatusException(message)); + return; + } + + communicator.forecastJob(params, (aVoid, e) -> { + if (e == null) { + handler.accept(null); + } else { + String msg = String.format(Locale.ROOT, "[%s] exception while forecasting job", jobTask.getJobId()); + logger.error(msg, e); + handler.accept(ExceptionsHelper.serverError(msg, e)); + } + }); + } + public void writeUpdateProcessMessage(JobTask jobTask, List updates, ModelPlotConfig config, Consumer handler) { AutodetectCommunicator communicator = autoDetectCommunicatorByOpenJob.get(jobTask.getAllocationId()); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/BlackHoleAutodetectProcess.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/BlackHoleAutodetectProcess.java index a9d091a5b82..5eba87765f6 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/BlackHoleAutodetectProcess.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/BlackHoleAutodetectProcess.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.ml.job.persistence.StateStreamer; import org.elasticsearch.xpack.ml.job.process.autodetect.output.FlushAcknowledgement; import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams; import org.elasticsearch.xpack.ml.job.process.autodetect.params.FlushJobParams; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.ForecastParams; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; import org.elasticsearch.xpack.ml.job.results.AutodetectResult; @@ -78,7 +79,7 @@ public class BlackHoleAutodetectProcess implements AutodetectProcess { @Override public String flushJob(FlushJobParams params) throws IOException { FlushAcknowledgement flushAcknowledgement = new FlushAcknowledgement(FLUSH_ID, null); - AutodetectResult result = new AutodetectResult(null, null, null, null, null, null, null, null, flushAcknowledgement); + AutodetectResult result = new AutodetectResult(null, null, null, null, null, null, null, null, null, null, flushAcknowledgement); results.add(result); return FLUSH_ID; } @@ -91,7 +92,7 @@ public class BlackHoleAutodetectProcess implements AutodetectProcess { public void close() throws IOException { if (open) { Quantiles quantiles = new Quantiles(jobId, new Date(), "black hole quantiles"); - AutodetectResult result = new AutodetectResult(null, null, null, quantiles, null, null, null, null, null); + AutodetectResult result = new AutodetectResult(null, null, null, quantiles, null, null, null, null, null, null, null); results.add(result); open = false; } @@ -147,4 +148,8 @@ public class BlackHoleAutodetectProcess implements AutodetectProcess { public String readError() { return ""; } + + @Override + public void forecastJob(ForecastParams params) throws IOException { + } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java index 3c2bf1c5589..9cee3db760a 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java @@ -17,6 +17,7 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutodetectResult import org.elasticsearch.xpack.ml.job.process.autodetect.output.StateProcessor; import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams; import org.elasticsearch.xpack.ml.job.process.autodetect.params.FlushJobParams; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.ForecastParams; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.ml.job.process.autodetect.writer.ControlMsgToProcessWriter; import org.elasticsearch.xpack.ml.job.process.autodetect.writer.LengthEncodedWriter; @@ -94,7 +95,9 @@ class NativeAutodetectProcess implements AutodetectProcess { if (processCloseInitiated == false && processKilled == false) { // The log message doesn't say "crashed", as the process could have been killed // by a user or other process (e.g. the Linux OOM killer) - LOGGER.error("[{}] autodetect process stopped unexpectedly", jobId); + + String errors = cppLogHandler.getErrors(); + LOGGER.error("[{}] autodetect process stopped unexpectedly: {}", jobId, errors); onProcessCrash.run(); } } @@ -163,6 +166,12 @@ class NativeAutodetectProcess implements AutodetectProcess { return writer.writeFlushMessage(); } + @Override + public void forecastJob(ForecastParams params) throws IOException { + ControlMsgToProcessWriter writer = new ControlMsgToProcessWriter(recordWriter, numberOfAnalysisFields); + writer.writeForecastMessage(params); + } + @Override public void flushStream() throws IOException { recordWriter.flush(); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java index 704b6d6082a..2a907a0c652 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java @@ -26,6 +26,8 @@ import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; import org.elasticsearch.xpack.ml.job.results.AutodetectResult; import org.elasticsearch.xpack.ml.job.results.Bucket; import org.elasticsearch.xpack.ml.job.results.CategoryDefinition; +import org.elasticsearch.xpack.ml.job.results.Forecast; +import org.elasticsearch.xpack.ml.job.results.ForecastStats; import org.elasticsearch.xpack.ml.job.results.Influencer; import org.elasticsearch.xpack.ml.job.results.ModelPlot; @@ -184,6 +186,20 @@ public class AutoDetectResultProcessor { if (modelPlot != null) { context.bulkResultsPersister.persistModelPlot(modelPlot); } + Forecast forecast = result.getForecast(); + if (forecast != null) { + context.bulkResultsPersister.persistForecast(forecast); + } + ForecastStats forecastStats = result.getForecastStats(); + if (forecastStats != null) { + // forecast stats are send by autodetect but do not get persisted, + // still they mark the end of a forecast + + LOGGER.trace("Received Forecast Stats [{}]", forecastStats.getId()); + + // forecast stats mark the end of a forecast, therefore commit whatever we have + context.bulkResultsPersister.executeRequest(); + } ModelSizeStats modelSizeStats = result.getModelSizeStats(); if (modelSizeStats != null) { LOGGER.trace("[{}] Parsed ModelSizeStats: {} / {} / {} / {} / {} / {}", diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/ForecastParams.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/ForecastParams.java new file mode 100644 index 00000000000..d1e3f38d23d --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/ForecastParams.java @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.params; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.joda.DateMathParser; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.xpack.ml.job.messages.Messages; + +import java.util.Objects; + +public class ForecastParams { + + private final long endTime; + private final long id; + + private ForecastParams(long id, long endTime) { + this.id = id; + this.endTime = endTime; + } + + /** + * The forecast end time in seconds from the epoch + * @return The end time in seconds from the epoch + */ + public long getEndTime() { + return endTime; + } + + /** + * The forecast id + * + * @return The forecast Id + */ + public long getId() { + return id; + } + + @Override + public int hashCode() { + return Objects.hash(id, endTime); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + ForecastParams other = (ForecastParams) obj; + return Objects.equals(id, other.id) && Objects.equals(endTime, other.endTime); + } + + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private long endTimeEpochSecs; + private long startTime; + private long forecastId; + + private Builder() { + startTime = System.currentTimeMillis(); + endTimeEpochSecs = tomorrow(startTime); + forecastId = generateId(); + } + + static long tomorrow(long now) { + return (now / 1000) + (60 * 60 * 24); + } + + private long generateId() { + return startTime; + } + + public Builder endTime(String endTime, ParseField paramName) { + DateMathParser dateMathParser = new DateMathParser(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER); + + try { + endTimeEpochSecs = dateMathParser.parse(endTime, System::currentTimeMillis) / 1000; + } catch (Exception e) { + String msg = Messages.getMessage(Messages.REST_INVALID_DATETIME_PARAMS, paramName.getPreferredName(), endTime); + throw new ElasticsearchParseException(msg, e); + } + + return this; + } + + public ForecastParams build() { + return new ForecastParams(forecastId, endTimeEpochSecs); + } + } +} + diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriter.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriter.java index a43b26890ec..a174c528330 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriter.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriter.java @@ -7,11 +7,13 @@ package org.elasticsearch.xpack.ml.job.process.autodetect.writer; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.xpack.ml.job.config.DetectionRule; import org.elasticsearch.xpack.ml.job.config.ModelPlotConfig; import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams; import org.elasticsearch.xpack.ml.job.process.autodetect.params.FlushJobParams; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.ForecastParams; import java.io.IOException; import java.io.OutputStream; @@ -37,6 +39,11 @@ public class ControlMsgToProcessWriter { */ private static final String FLUSH_MESSAGE_CODE = "f"; + /** + * This must match the code defined in the api::CAnomalyDetector C++ class. + */ + private static final String FORECAST_MESSAGE_CODE = "p"; + /** * This must match the code defined in the api::CAnomalyDetector C++ class. */ @@ -137,14 +144,32 @@ public class ControlMsgToProcessWriter { String flushId = Long.toString(ms_FlushNumber.getAndIncrement()); writeMessage(FLUSH_MESSAGE_CODE + flushId); - char[] spaces = new char[FLUSH_SPACES_LENGTH]; - Arrays.fill(spaces, ' '); - writeMessage(new String(spaces)); + fillCommandBuffer(); lengthEncodedWriter.flush(); return flushId; } + public void writeForecastMessage(ForecastParams params) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .field("forecast_id", params.getId()) + .field("end_time", params.getEndTime()) + .endObject(); + + writeMessage(FORECAST_MESSAGE_CODE + builder.string()); + fillCommandBuffer(); + lengthEncodedWriter.flush(); + } + + // todo(hendrikm): workaround, see + // https://github.com/elastic/machine-learning-cpp/issues/123 + private void fillCommandBuffer() throws IOException { + char[] spaces = new char[FLUSH_SPACES_LENGTH]; + Arrays.fill(spaces, ' '); + writeMessage(new String(spaces)); + } + public void writeResetBucketsMessage(DataLoadParams params) throws IOException { writeControlCodeFollowedByTimeRange(RESET_BUCKETS_MESSAGE_CODE, params.getStart(), params.getEnd()); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/AutodetectResult.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/AutodetectResult.java index c572e8dd434..be8a5841690 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/AutodetectResult.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/AutodetectResult.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.ml.job.results; +import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -31,7 +32,7 @@ public class AutodetectResult implements ToXContentObject, Writeable { TYPE.getPreferredName(), a -> new AutodetectResult((Bucket) a[0], (List) a[1], (List) a[2], (Quantiles) a[3], a[4] == null ? null : ((ModelSnapshot.Builder) a[4]).build(), a[5] == null ? null : ((ModelSizeStats.Builder) a[5]).build(), - (ModelPlot) a[6], (CategoryDefinition) a[7], (FlushAcknowledgement) a[8])); + (ModelPlot) a[6], (Forecast) a[7], (ForecastStats) a[8], (CategoryDefinition) a[9], (FlushAcknowledgement) a[10])); static { PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Bucket.PARSER, Bucket.RESULT_TYPE_FIELD); @@ -42,6 +43,8 @@ public class AutodetectResult implements ToXContentObject, Writeable { PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ModelSizeStats.PARSER, ModelSizeStats.RESULT_TYPE_FIELD); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ModelPlot.PARSER, ModelPlot.RESULTS_FIELD); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Forecast.PARSER, Forecast.RESULTS_FIELD); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ForecastStats.PARSER, ForecastStats.RESULTS_FIELD); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), CategoryDefinition.PARSER, CategoryDefinition.TYPE); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), FlushAcknowledgement.PARSER, FlushAcknowledgement.TYPE); } @@ -53,12 +56,14 @@ public class AutodetectResult implements ToXContentObject, Writeable { private final ModelSnapshot modelSnapshot; private final ModelSizeStats modelSizeStats; private final ModelPlot modelPlot; + private final Forecast forecast; + private final ForecastStats forecastStats; private final CategoryDefinition categoryDefinition; private final FlushAcknowledgement flushAcknowledgement; public AutodetectResult(Bucket bucket, List records, List influencers, Quantiles quantiles, - ModelSnapshot modelSnapshot, ModelSizeStats modelSizeStats, ModelPlot modelPlot, - CategoryDefinition categoryDefinition, FlushAcknowledgement flushAcknowledgement) { + ModelSnapshot modelSnapshot, ModelSizeStats modelSizeStats, ModelPlot modelPlot, Forecast forecast, + ForecastStats forecastStats, CategoryDefinition categoryDefinition, FlushAcknowledgement flushAcknowledgement) { this.bucket = bucket; this.records = records; this.influencers = influencers; @@ -66,6 +71,8 @@ public class AutodetectResult implements ToXContentObject, Writeable { this.modelSnapshot = modelSnapshot; this.modelSizeStats = modelSizeStats; this.modelPlot = modelPlot; + this.forecast = forecast; + this.forecastStats = forecastStats; this.categoryDefinition = categoryDefinition; this.flushAcknowledgement = flushAcknowledgement; } @@ -116,6 +123,22 @@ public class AutodetectResult implements ToXContentObject, Writeable { } else { this.flushAcknowledgement = null; } + + if (in.getVersion().onOrAfter(Version.V_6_1_0)) { + if (in.readBoolean()) { + this.forecast = new Forecast(in); + } else { + this.forecast = null; + } + if (in.readBoolean()) { + this.forecastStats = new ForecastStats(in); + } else { + this.forecastStats = null; + } + } else { + this.forecast = null; + this.forecastStats = null; + } } @Override @@ -129,6 +152,11 @@ public class AutodetectResult implements ToXContentObject, Writeable { writeNullable(modelPlot, out); writeNullable(categoryDefinition, out); writeNullable(flushAcknowledgement, out); + + if (out.getVersion().onOrAfter(Version.V_6_1_0)) { + writeNullable(forecast, out); + writeNullable(forecastStats, out); + } } private static void writeNullable(Writeable writeable, StreamOutput out) throws IOException { @@ -157,6 +185,8 @@ public class AutodetectResult implements ToXContentObject, Writeable { addNullableField(ModelSnapshot.TYPE, modelSnapshot, builder); addNullableField(ModelSizeStats.RESULT_TYPE_FIELD, modelSizeStats, builder); addNullableField(ModelPlot.RESULTS_FIELD, modelPlot, builder); + addNullableField(Forecast.RESULTS_FIELD, forecast, builder); + addNullableField(ForecastStats.RESULTS_FIELD, forecastStats, builder); addNullableField(CategoryDefinition.TYPE, categoryDefinition, builder); addNullableField(FlushAcknowledgement.TYPE, flushAcknowledgement, builder); builder.endObject(); @@ -203,6 +233,14 @@ public class AutodetectResult implements ToXContentObject, Writeable { return modelPlot; } + public Forecast getForecast() { + return forecast; + } + + public ForecastStats getForecastStats() { + return forecastStats; + } + public CategoryDefinition getCategoryDefinition() { return categoryDefinition; } @@ -213,8 +251,8 @@ public class AutodetectResult implements ToXContentObject, Writeable { @Override public int hashCode() { - return Objects.hash(bucket, records, influencers, categoryDefinition, flushAcknowledgement, modelPlot, modelSizeStats, - modelSnapshot, quantiles); + return Objects.hash(bucket, records, influencers, categoryDefinition, flushAcknowledgement, modelPlot, forecast, forecastStats, + modelSizeStats, modelSnapshot, quantiles); } @Override @@ -232,6 +270,8 @@ public class AutodetectResult implements ToXContentObject, Writeable { Objects.equals(categoryDefinition, other.categoryDefinition) && Objects.equals(flushAcknowledgement, other.flushAcknowledgement) && Objects.equals(modelPlot, other.modelPlot) && + Objects.equals(forecast, other.forecast) && + Objects.equals(forecastStats, other.forecastStats) && Objects.equals(modelSizeStats, other.modelSizeStats) && Objects.equals(modelSnapshot, other.modelSnapshot) && Objects.equals(quantiles, other.quantiles); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/Forecast.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/Forecast.java new file mode 100644 index 00000000000..3a558fbffff --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/Forecast.java @@ -0,0 +1,308 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.results; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +/** + * Model Forecast POJO. + */ +public class Forecast implements ToXContentObject, Writeable { + /** + * Result type + */ + public static final String RESULT_TYPE_VALUE = "model_forecast"; + public static final ParseField RESULTS_FIELD = new ParseField(RESULT_TYPE_VALUE); + + public static final ParseField FORECAST_ID = new ParseField("forecast_id"); + public static final ParseField PARTITION_FIELD_NAME = new ParseField("partition_field_name"); + public static final ParseField PARTITION_FIELD_VALUE = new ParseField("partition_field_value"); + public static final ParseField OVER_FIELD_NAME = new ParseField("over_field_name"); + public static final ParseField OVER_FIELD_VALUE = new ParseField("over_field_value"); + public static final ParseField BY_FIELD_NAME = new ParseField("by_field_name"); + public static final ParseField BY_FIELD_VALUE = new ParseField("by_field_value"); + public static final ParseField MODEL_FEATURE = new ParseField("model_feature"); + public static final ParseField FORECAST_LOWER = new ParseField("forecast_lower"); + public static final ParseField FORECAST_UPPER = new ParseField("forecast_upper"); + public static final ParseField FORECAST_PREDICTION = new ParseField("forecast_prediction"); + public static final ParseField BUCKET_SPAN = new ParseField("bucket_span"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(RESULT_TYPE_VALUE, a -> new Forecast((String) a[0], (long) a[1], (Date) a[2], (long) a[3])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), FORECAST_ID); + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + + Result.TIMESTAMP.getPreferredName() + "]"); + }, Result.TIMESTAMP, ValueType.VALUE); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); + PARSER.declareString((modelForecast, s) -> {}, Result.RESULT_TYPE); + PARSER.declareString(Forecast::setPartitionFieldName, PARTITION_FIELD_NAME); + PARSER.declareString(Forecast::setPartitionFieldValue, PARTITION_FIELD_VALUE); + PARSER.declareString(Forecast::setOverFieldName, OVER_FIELD_NAME); + PARSER.declareString(Forecast::setOverFieldValue, OVER_FIELD_VALUE); + PARSER.declareString(Forecast::setByFieldName, BY_FIELD_NAME); + PARSER.declareString(Forecast::setByFieldValue, BY_FIELD_VALUE); + PARSER.declareString(Forecast::setModelFeature, MODEL_FEATURE); + PARSER.declareDouble(Forecast::setForecastLower, FORECAST_LOWER); + PARSER.declareDouble(Forecast::setForecastUpper, FORECAST_UPPER); + PARSER.declareDouble(Forecast::setForecastPrediction, FORECAST_PREDICTION); + } + + private final String jobId; + private final long forecastId; + private final Date timestamp; + private final long bucketSpan; + private String partitionFieldName; + private String partitionFieldValue; + private String overFieldName; + private String overFieldValue; + private String byFieldName; + private String byFieldValue; + private String modelFeature; + private double forecastLower; + private double forecastUpper; + private double forecastPrediction; + + public Forecast(String jobId, long forecastId, Date timestamp, long bucketSpan) { + this.jobId = jobId; + this.forecastId = forecastId; + this.timestamp = timestamp; + this.bucketSpan = bucketSpan; + } + + public Forecast(StreamInput in) throws IOException { + jobId = in.readString(); + forecastId = in.readLong(); + timestamp = new Date(in.readLong()); + partitionFieldName = in.readOptionalString(); + partitionFieldValue = in.readOptionalString(); + overFieldName = in.readOptionalString(); + overFieldValue = in.readOptionalString(); + byFieldName = in.readOptionalString(); + byFieldValue = in.readOptionalString(); + modelFeature = in.readOptionalString(); + forecastLower = in.readDouble(); + forecastUpper = in.readDouble(); + forecastPrediction = in.readDouble(); + bucketSpan = in.readLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeLong(forecastId); + out.writeLong(timestamp.getTime()); + out.writeOptionalString(partitionFieldName); + out.writeOptionalString(partitionFieldValue); + out.writeOptionalString(overFieldName); + out.writeOptionalString(overFieldValue); + out.writeOptionalString(byFieldName); + out.writeOptionalString(byFieldValue); + out.writeOptionalString(modelFeature); + out.writeDouble(forecastLower); + out.writeDouble(forecastUpper); + out.writeDouble(forecastPrediction); + out.writeLong(bucketSpan); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(FORECAST_ID.getPreferredName(), forecastId); + builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE); + builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); + if (timestamp != null) { + builder.dateField(Result.TIMESTAMP.getPreferredName(), + Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); + } + if (partitionFieldName != null) { + builder.field(PARTITION_FIELD_NAME.getPreferredName(), partitionFieldName); + } + if (partitionFieldValue != null) { + builder.field(PARTITION_FIELD_VALUE.getPreferredName(), partitionFieldValue); + } + if (overFieldName != null) { + builder.field(OVER_FIELD_NAME.getPreferredName(), overFieldName); + } + if (overFieldValue != null) { + builder.field(OVER_FIELD_VALUE.getPreferredName(), overFieldValue); + } + if (byFieldName != null) { + builder.field(BY_FIELD_NAME.getPreferredName(), byFieldName); + } + if (byFieldValue != null) { + builder.field(BY_FIELD_VALUE.getPreferredName(), byFieldValue); + } + if (modelFeature != null) { + builder.field(MODEL_FEATURE.getPreferredName(), modelFeature); + } + builder.field(FORECAST_LOWER.getPreferredName(), forecastLower); + builder.field(FORECAST_UPPER.getPreferredName(), forecastUpper); + builder.field(FORECAST_PREDICTION.getPreferredName(), forecastPrediction); + builder.endObject(); + return builder; + } + + public String getJobId() { + return jobId; + } + + public long getForecastId() { + return forecastId; + } + + public String getId() { + int valuesHash = Objects.hash(byFieldValue, overFieldValue, partitionFieldValue); + int length = (byFieldValue == null ? 0 : byFieldValue.length()) + + (overFieldValue == null ? 0 : overFieldValue.length()) + + (partitionFieldValue == null ? 0 : partitionFieldValue.length()); + return jobId + "_model_forecast_" + forecastId + "_" + timestamp.getTime() + "_" + bucketSpan + "_" + + (modelFeature == null ? "" : modelFeature) + "_" + valuesHash + "_" + length; + } + + public Date getTimestamp() { + return timestamp; + } + + public long getBucketSpan() { + return bucketSpan; + } + + public String getPartitionFieldName() { + return partitionFieldName; + } + + public void setPartitionFieldName(String partitionFieldName) { + this.partitionFieldName = partitionFieldName; + } + + public String getPartitionFieldValue() { + return partitionFieldValue; + } + + public void setPartitionFieldValue(String partitionFieldValue) { + this.partitionFieldValue = partitionFieldValue; + } + + public String getOverFieldName() { + return overFieldName; + } + + public void setOverFieldName(String overFieldName) { + this.overFieldName = overFieldName; + } + + public String getOverFieldValue() { + return overFieldValue; + } + + public void setOverFieldValue(String overFieldValue) { + this.overFieldValue = overFieldValue; + } + + public String getByFieldName() { + return byFieldName; + } + + public void setByFieldName(String byFieldName) { + this.byFieldName = byFieldName; + } + + public String getByFieldValue() { + return byFieldValue; + } + + public void setByFieldValue(String byFieldValue) { + this.byFieldValue = byFieldValue; + } + + public String getModelFeature() { + return modelFeature; + } + + public void setModelFeature(String modelFeature) { + this.modelFeature = modelFeature; + } + + public double getForecastLower() { + return forecastLower; + } + + public void setForecastLower(double forecastLower) { + this.forecastLower = forecastLower; + } + + public double getForecastUpper() { + return forecastUpper; + } + + public void setForecastUpper(double forecastUpper) { + this.forecastUpper = forecastUpper; + } + + public double getForecastPrediction() { + return forecastPrediction; + } + + public void setForecastPrediction(double forecastPrediction) { + this.forecastPrediction = forecastPrediction; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other instanceof Forecast == false) { + return false; + } + Forecast that = (Forecast) other; + return Objects.equals(this.jobId, that.jobId) && + forecastId == that.forecastId && + Objects.equals(this.timestamp, that.timestamp) && + Objects.equals(this.partitionFieldValue, that.partitionFieldValue) && + Objects.equals(this.partitionFieldName, that.partitionFieldName) && + Objects.equals(this.overFieldValue, that.overFieldValue) && + Objects.equals(this.overFieldName, that.overFieldName) && + Objects.equals(this.byFieldValue, that.byFieldValue) && + Objects.equals(this.byFieldName, that.byFieldName) && + Objects.equals(this.modelFeature, that.modelFeature) && + this.forecastLower == that.forecastLower && + this.forecastUpper == that.forecastUpper && + this.forecastPrediction == that.forecastPrediction && + this.bucketSpan == that.bucketSpan; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, forecastId, timestamp, partitionFieldName, partitionFieldValue, + overFieldName, overFieldValue, byFieldName, byFieldValue, + modelFeature, forecastLower, forecastUpper, forecastPrediction, bucketSpan); + } +} diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ForecastStats.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ForecastStats.java new file mode 100644 index 00000000000..9ec3122bc69 --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ForecastStats.java @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.results; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; +import java.util.Objects; + +/** + * Model ForecastStats POJO. + * + * Note forecast stats are sent from the autodetect process but do not get + * indexed. + */ +public class ForecastStats implements ToXContentObject, Writeable { + /** + * Result type + */ + public static final String RESULT_TYPE_VALUE = "model_forecast_stats"; + public static final ParseField RESULTS_FIELD = new ParseField(RESULT_TYPE_VALUE); + + public static final ParseField FORECAST_ID = new ParseField("forecast_id"); + public static final ParseField RECORD_COUNT = new ParseField("forecast_record_count"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(RESULT_TYPE_VALUE, a -> new ForecastStats((String) a[0], (long) a[1])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), FORECAST_ID); + + PARSER.declareString((modelForecastStats, s) -> {}, Result.RESULT_TYPE); + PARSER.declareLong(ForecastStats::setRecordCount, RECORD_COUNT); + } + + private final String jobId; + private final long forecastId; + private long recordCount; + + public ForecastStats(String jobId, long forecastId) { + this.jobId = jobId; + this.forecastId = forecastId; + } + + public ForecastStats(StreamInput in) throws IOException { + jobId = in.readString(); + forecastId = in.readLong(); + recordCount = in.readLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeLong(forecastId); + out.writeLong(recordCount); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE); + builder.field(FORECAST_ID.getPreferredName(), forecastId); + builder.field(RECORD_COUNT.getPreferredName(), recordCount); + builder.endObject(); + return builder; + } + + public String getJobId() { + return jobId; + } + + public String getId() { + return jobId + "_model_forecast_stats"; + } + + public void setRecordCount(long recordCount) { + this.recordCount = recordCount; + } + + public double getRecordCount() { + return recordCount; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other instanceof ForecastStats == false) { + return false; + } + ForecastStats that = (ForecastStats) other; + return Objects.equals(this.jobId, that.jobId) && + this.forecastId == that.forecastId && + this.recordCount == that.recordCount; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, forecastId, recordCount); + } +} diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java index b4404e931f4..565951b20d3 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java @@ -127,6 +127,10 @@ public final class ReservedFieldNames { ModelPlot.MODEL_UPPER.getPreferredName(), ModelPlot.MODEL_MEDIAN.getPreferredName(), ModelPlot.ACTUAL.getPreferredName(), + Forecast.FORECAST_LOWER.getPreferredName(), Forecast.FORECAST_UPPER.getPreferredName(), + Forecast.FORECAST_PREDICTION.getPreferredName(), + Forecast.FORECAST_ID.getPreferredName(), + ModelSizeStats.MODEL_BYTES_FIELD.getPreferredName(), ModelSizeStats.TOTAL_BY_FIELD_COUNT_FIELD.getPreferredName(), ModelSizeStats.TOTAL_OVER_FIELD_COUNT_FIELD.getPreferredName(), diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestForecastJobAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestForecastJobAction.java new file mode 100644 index 00000000000..7f80c1cce52 --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestForecastJobAction.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.job; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.action.ForecastJobAction; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; + +public class RestForecastJobAction extends BaseRestHandler { + + public RestForecastJobAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.POST, + MachineLearning.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/_forecast", this); + } + + @Override + public String getName() { + return "xpack_ml_forecast_job_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String jobId = restRequest.param(Job.ID.getPreferredName()); + final ForecastJobAction.Request request; + if (restRequest.hasContentOrSourceParam()) { + XContentParser parser = restRequest.contentOrSourceParamParser(); + request = ForecastJobAction.Request.parseRequest(jobId, parser); + } else { + request = new ForecastJobAction.Request(restRequest.param(Job.ID.getPreferredName())); + if (restRequest.hasParam(ForecastJobAction.Request.END_TIME.getPreferredName())) { + request.setEndTime(restRequest.param(ForecastJobAction.Request.END_TIME.getPreferredName())); + } + } + + return channel -> client.execute(ForecastJobAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } +} diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/ForecastJobActionRequestTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/ForecastJobActionRequestTests.java new file mode 100644 index 00000000000..bc143b84406 --- /dev/null +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/ForecastJobActionRequestTests.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractStreamableXContentTestCase; +import org.elasticsearch.xpack.ml.action.ForecastJobAction.Request; + +public class ForecastJobActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request doParseInstance(XContentParser parser) { + return Request.parseRequest(null, parser); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } + + @Override + protected Request createTestInstance() { + Request request = new Request(randomAlphaOfLengthBetween(1, 20)); + return request; + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/ForecastJobActionResponseTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/ForecastJobActionResponseTests.java new file mode 100644 index 00000000000..7933b49f3fd --- /dev/null +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/ForecastJobActionResponseTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.xpack.ml.action.ForecastJobAction.Response; + +public class ForecastJobActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + return new Response(randomBoolean(), randomNonNegativeLong()); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java index 5b7fbe3826f..35b7e28bb35 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java @@ -357,47 +357,47 @@ public class AutodetectResultProcessorIT extends XPackSingleNodeTestCase { private List results = new ArrayList<>(); ResultsBuilder addBucket(Bucket bucket) { - results.add(new AutodetectResult(Objects.requireNonNull(bucket), null, null, null, null, null, null, null, null)); + results.add(new AutodetectResult(Objects.requireNonNull(bucket), null, null, null, null, null, null, null, null, null, null)); return this; } ResultsBuilder addRecords(List records) { - results.add(new AutodetectResult(null, records, null, null, null, null, null, null, null)); + results.add(new AutodetectResult(null, records, null, null, null, null, null, null, null, null, null)); return this; } ResultsBuilder addInfluencers(List influencers) { - results.add(new AutodetectResult(null, null, influencers, null, null, null, null, null, null)); + results.add(new AutodetectResult(null, null, influencers, null, null, null, null, null, null, null, null)); return this; } ResultsBuilder addCategoryDefinition(CategoryDefinition categoryDefinition) { - results.add(new AutodetectResult(null, null, null, null, null, null, null, categoryDefinition, null)); + results.add(new AutodetectResult(null, null, null, null, null, null, null, null, null, categoryDefinition, null)); return this; } ResultsBuilder addmodelPlot(ModelPlot modelPlot) { - results.add(new AutodetectResult(null, null, null, null, null, null, modelPlot, null, null)); + results.add(new AutodetectResult(null, null, null, null, null, null, modelPlot, null, null, null, null)); return this; } ResultsBuilder addModelSizeStats(ModelSizeStats modelSizeStats) { - results.add(new AutodetectResult(null, null, null, null, null, modelSizeStats, null, null, null)); + results.add(new AutodetectResult(null, null, null, null, null, modelSizeStats, null, null, null, null, null)); return this; } ResultsBuilder addModelSnapshot(ModelSnapshot modelSnapshot) { - results.add(new AutodetectResult(null, null, null, null, modelSnapshot, null, null, null, null)); + results.add(new AutodetectResult(null, null, null, null, modelSnapshot, null, null, null, null, null, null)); return this; } ResultsBuilder addQuantiles(Quantiles quantiles) { - results.add(new AutodetectResult(null, null, null, quantiles, null, null, null, null, null)); + results.add(new AutodetectResult(null, null, null, quantiles, null, null, null, null, null, null, null)); return this; } ResultsBuilder addFlushAcknowledgement(FlushAcknowledgement flushAcknowledgement) { - results.add(new AutodetectResult(null, null, null, null, null, null, null, null, flushAcknowledgement)); + results.add(new AutodetectResult(null, null, null, null, null, null, null, null, null, null, flushAcknowledgement)); return this; } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/ForecastParamsTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/ForecastParamsTests.java new file mode 100644 index 00000000000..f2b59d6a103 --- /dev/null +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/ForecastParamsTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.params; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.job.messages.Messages; + +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +public class ForecastParamsTests extends ESTestCase { + + private static ParseField END = new ParseField("end"); + + public void testDefault_GivesTomorrowTimeInSeconds() { + long nowSecs = System.currentTimeMillis() / 1000; + nowSecs += 60 * 60 * 24; + + ForecastParams params = ForecastParams.builder().build(); + assertThat(params.getEndTime(), greaterThanOrEqualTo(nowSecs)); + assertThat(params.getEndTime(), lessThanOrEqualTo(nowSecs +1)); + } + + public void test_UnparseableEndTimeThrows() { + ElasticsearchParseException e = + ESTestCase.expectThrows(ElasticsearchParseException.class, () -> ForecastParams.builder().endTime("bad", END).build()); + assertEquals(Messages.getMessage(Messages.REST_INVALID_DATETIME_PARAMS, "end", "bad"), e.getMessage()); + } + + public void testFormats() { + assertEquals(10L, ForecastParams.builder().endTime("10000", END).build().getEndTime()); + assertEquals(1462096800L, ForecastParams.builder().endTime("2016-05-01T10:00:00Z", END).build().getEndTime()); + + long nowSecs = System.currentTimeMillis() / 1000; + long end = ForecastParams.builder().endTime("now+2H", END).build().getEndTime(); + assertThat(end, greaterThanOrEqualTo(nowSecs + 7200)); + assertThat(end, lessThanOrEqualTo(nowSecs + 7200 +1)); + } +} diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/AutodetectResultTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/AutodetectResultTests.java index 1f9cb78f590..2090275f625 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/AutodetectResultTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/AutodetectResultTests.java @@ -35,6 +35,8 @@ public class AutodetectResultTests extends AbstractSerializingTestCase { + + @Override + protected ForecastStats parseInstance(XContentParser parser) { + return ForecastStats.PARSER.apply(parser, null); + } + + @Override + protected ForecastStats createTestInstance() { + return createTestInstance("ForecastStatsTest", randomNonNegativeLong()); + } + + public ForecastStats createTestInstance(String jobId, long forecastId) { + ForecastStats forecastStats = new ForecastStats(jobId, forecastId); + + if (randomBoolean()) { + forecastStats.setRecordCount(randomLong()); + } + + return forecastStats; + } + + @Override + protected Reader instanceReader() { + return ForecastStats::new; + } + + @Override + protected ForecastStats doParseInstance(XContentParser parser) throws IOException { + return ForecastStats.PARSER.apply(parser, null); + } + +} diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastTests.java new file mode 100644 index 00000000000..97249ae4f76 --- /dev/null +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastTests.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.results; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; + +import java.io.IOException; +import java.util.Date; + +public class ForecastTests extends AbstractSerializingTestCase { + + @Override + protected Forecast parseInstance(XContentParser parser) { + return Forecast.PARSER.apply(parser, null); + } + + @Override + protected Forecast createTestInstance() { + return createTestInstance("ForecastTest"); + } + + public Forecast createTestInstance(String jobId) { + Forecast forecast = new Forecast(jobId, randomNonNegativeLong(), new Date(randomLong()), randomNonNegativeLong()); + + if (randomBoolean()) { + forecast.setByFieldName(randomAlphaOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + forecast.setByFieldValue(randomAlphaOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + forecast.setPartitionFieldName(randomAlphaOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + forecast.setPartitionFieldValue(randomAlphaOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + forecast.setModelFeature(randomAlphaOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + forecast.setForecastLower(randomDouble()); + } + if (randomBoolean()) { + forecast.setForecastUpper(randomDouble()); + } + if (randomBoolean()) { + forecast.setForecastPrediction(randomDouble()); + } + + return forecast; + } + + @Override + protected Reader instanceReader() { + return Forecast::new; + } + + @Override + protected Forecast doParseInstance(XContentParser parser) throws IOException { + return Forecast.PARSER.apply(parser, null); + } + +} diff --git a/plugin/src/test/resources/org/elasticsearch/transport/actions b/plugin/src/test/resources/org/elasticsearch/transport/actions index ef4937816c5..496745bd1c7 100644 --- a/plugin/src/test/resources/org/elasticsearch/transport/actions +++ b/plugin/src/test/resources/org/elasticsearch/transport/actions @@ -157,3 +157,4 @@ indices:data/write/update/byquery indices:data/write/delete/byquery indices:data/write/reindex cluster:admin/xpack/deprecation/info +cluster:admin/xpack/ml/job/forecast diff --git a/plugin/src/test/resources/rest-api-spec/api/xpack.ml.forecast.json b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.forecast.json new file mode 100644 index 00000000000..04db5e2ecf2 --- /dev/null +++ b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.forecast.json @@ -0,0 +1,24 @@ +{ + "xpack.ml.forecast": { + "methods": [ "POST" ], + "url": { + "path": "/_xpack/ml/anomaly_detectors/{job_id}/_forecast", + "paths": [ "/_xpack/ml/anomaly_detectors/{job_id}/_forecast" ], + "parts": { + "job_id": { + "type": "string", + "required": true, + "description": "The ID of the job to forecast for" + } + }, + "params": { + "end": { + "type": "string", + "required": false, + "description": "The end time of the forecast" + } + } + }, + "body": null + } +} diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/forecast.yml b/plugin/src/test/resources/rest-api-spec/test/ml/forecast.yml new file mode 100644 index 00000000000..643af83a78f --- /dev/null +++ b/plugin/src/test/resources/rest-api-spec/test/ml/forecast.yml @@ -0,0 +1,41 @@ +setup: + - do: + xpack.ml.put_job: + job_id: forecast-job + body: > + { + "description":"A forecast job", + "analysis_config" : { + "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}] + }, + "data_description" : { + "format":"xcontent" + } + } + +--- +"Test forecast unknown job": + - do: + catch: missing + xpack.ml.forecast: + job_id: "non-existing-job" + +--- +"Test forecast on closed job": + - do: + catch: /status_exception/ + xpack.ml.forecast: + job_id: "forecast-job" + +--- +"Test bad end param errors": + + - do: + xpack.ml.open_job: + job_id: "forecast-job" + + - do: + catch: /parse_exception/ + xpack.ml.forecast: + job_id: "forecast-job" + end: "tomorrow" From 104a3a323f836bde39591df15fd3cfc48d1225f2 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Thu, 14 Sep 2017 14:46:48 +0100 Subject: [PATCH 6/7] [BUILD] Make AWS error message more informative Original commit: elastic/x-pack-elasticsearch@42cca7ed828b0aae923a3e2fbaeb4be9cd3a23db --- plugin/ml-cpp-snapshot/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/ml-cpp-snapshot/build.gradle b/plugin/ml-cpp-snapshot/build.gradle index 860e1d5ddf9..f37091c1945 100644 --- a/plugin/ml-cpp-snapshot/build.gradle +++ b/plugin/ml-cpp-snapshot/build.gradle @@ -111,7 +111,7 @@ S3Object getZip() { return client.getObject('prelert-artifacts', key) } catch (AmazonServiceException e) { if (e.getStatusCode() != 403) { - throw new GradleException('Error while trying to get ml-cpp snapshot', e) + throw new GradleException('Error while trying to get ml-cpp snapshot: ' + e.getMessage(), e) } sleep(500) retries-- From 8d8baffe2421c676ce57092ffe513cbe96bfde86 Mon Sep 17 00:00:00 2001 From: Jay Modi Date: Thu, 14 Sep 2017 08:09:14 -0600 Subject: [PATCH 7/7] Add specific client and user for security index access (elastic/x-pack-elasticsearch#2492) This change removes security index access from the xpack user by creating its own specific role and adds a xpack security user that maintains the superuser role so that it can perform all operations necessary for security. Original commit: elastic/x-pack-elasticsearch@ad906bc9135771101845c775658f3c0691919233 --- .../org/elasticsearch/xpack/XPackPlugin.java | 4 +- .../xpack/security/InternalClient.java | 9 ++- .../security/InternalSecurityClient.java | 23 +++++++ .../xpack/security/Security.java | 4 +- .../security/SecurityLifecycleService.java | 2 +- .../security/audit/index/IndexAuditTrail.java | 3 +- .../security/authc/ExpiredTokenRemover.java | 5 +- .../xpack/security/authc/TokenService.java | 5 +- .../authc/esnative/NativeUsersStore.java | 5 +- .../mapper/NativeRoleMappingStore.java | 5 +- .../security/authz/AuthorizationService.java | 8 ++- .../authz/store/NativeRolesStore.java | 5 +- .../authz/store/ReservedRolesStore.java | 3 +- .../support/IndexLifecycleManager.java | 5 +- .../xpack/security/user/User.java | 5 ++ .../security/user/XPackSecurityUser.java | 38 ++++++++++++ .../xpack/security/user/XPackUser.java | 13 +++- .../elasticsearch/xpack/upgrade/Upgrade.java | 7 ++- .../test/SecurityIntegTestCase.java | 6 ++ .../SecurityLifecycleServiceTests.java | 4 +- .../xpack/security/SecurityTests.java | 4 +- .../security/audit/index/AuditTrailTests.java | 2 +- .../index/IndexAuditTrailMutedTests.java | 5 +- .../audit/index/IndexAuditTrailTests.java | 2 +- .../IndexAuditTrailUpdateMappingTests.java | 2 +- .../authc/AuthenticationServiceTests.java | 3 +- .../security/authc/TokenAuthIntegTests.java | 6 +- .../security/authc/TokenServiceTests.java | 5 +- .../authc/esnative/NativeUsersStoreTests.java | 5 +- .../mapper/NativeUserRoleMapperTests.java | 3 +- .../authz/AuthorizationServiceTests.java | 61 ++++++++----------- .../authz/IndicesAndAliasesResolverTests.java | 14 ++++- .../authz/store/NativeRolesStoreTests.java | 3 +- .../authz/store/ReservedRolesStoreTests.java | 2 + .../support/IndexLifecycleManagerTests.java | 3 +- 35 files changed, 197 insertions(+), 82 deletions(-) create mode 100644 plugin/src/main/java/org/elasticsearch/xpack/security/InternalSecurityClient.java create mode 100644 plugin/src/main/java/org/elasticsearch/xpack/security/user/XPackSecurityUser.java diff --git a/plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java b/plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java index 8f6ffa1f78b..172c5683ad1 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java @@ -283,7 +283,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I components.add(licenseState); try { - components.addAll(security.createComponents(internalClient, threadPool, clusterService, resourceWatcherService, + components.addAll(security.createComponents(client, threadPool, clusterService, resourceWatcherService, extensionsService.getExtensions())); } catch (final Exception e) { throw new IllegalStateException("security initialization failed", e); @@ -319,7 +319,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I components.addAll(logstash.createComponents(internalClient, clusterService)); - components.addAll(upgrade.createComponents(internalClient, clusterService, threadPool, resourceWatcherService, + components.addAll(upgrade.createComponents(client, clusterService, threadPool, resourceWatcherService, scriptService, xContentRegistry)); // just create the reloader as it will pull all of the loaded ssl configurations and start watching them diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/InternalClient.java b/plugin/src/main/java/org/elasticsearch/xpack/security/InternalClient.java index d45c85bacb1..dd272c16fd0 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/InternalClient.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/InternalClient.java @@ -27,6 +27,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.XPackSettings; import org.elasticsearch.xpack.security.authc.Authentication; +import org.elasticsearch.xpack.security.user.User; import org.elasticsearch.xpack.security.user.XPackUser; import java.io.IOException; @@ -48,15 +49,21 @@ public class InternalClient extends FilterClient { private final String nodeName; private final boolean securityEnabled; + private final User user; /** * Constructs an InternalClient. * If security is enabled the client is secure. Otherwise this client is a passthrough. */ public InternalClient(Settings settings, ThreadPool threadPool, Client in) { + this(settings, threadPool, in, XPackUser.INSTANCE); + } + + InternalClient(Settings settings, ThreadPool threadPool, Client in, User user) { super(settings, threadPool, in); this.nodeName = Node.NODE_NAME_SETTING.get(settings); this.securityEnabled = XPackSettings.SECURITY_ENABLED.get(settings); + this.user = user; } @Override @@ -80,7 +87,7 @@ public class InternalClient extends FilterClient { protected void processContext(ThreadContext threadContext) { try { - Authentication authentication = new Authentication(XPackUser.INSTANCE, + Authentication authentication = new Authentication(user, new Authentication.RealmRef("__attach", "__attach", nodeName), null); authentication.writeToContext(threadContext); } catch (IOException ioe) { diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/InternalSecurityClient.java b/plugin/src/main/java/org/elasticsearch/xpack/security/InternalSecurityClient.java new file mode 100644 index 00000000000..e9e215615af --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/InternalSecurityClient.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.security; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.security.user.XPackSecurityUser; + +/** + * A special filter client for internal usage by security to modify the security index. + * + * The {@link XPackSecurityUser} user is added to the execution context before each action is executed. + */ +public class InternalSecurityClient extends InternalClient { + + public InternalSecurityClient(Settings settings, ThreadPool threadPool, Client in) { + super(settings, threadPool, in, XPackSecurityUser.INSTANCE); + } +} diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/Security.java b/plugin/src/main/java/org/elasticsearch/xpack/security/Security.java index dd9de5bc2ae..12b9fdc4379 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.action.support.DestructiveOperations; import org.elasticsearch.bootstrap.BootstrapCheck; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.LocalNodeMasterListener; import org.elasticsearch.cluster.NamedDiff; @@ -295,12 +296,13 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, Clus return modules; } - public Collection createComponents(InternalClient client, ThreadPool threadPool, ClusterService clusterService, + public Collection createComponents(Client nodeClient, ThreadPool threadPool, ClusterService clusterService, ResourceWatcherService resourceWatcherService, List extensions) throws Exception { if (enabled == false) { return Collections.emptyList(); } + final InternalSecurityClient client = new InternalSecurityClient(settings, threadPool, nodeClient); threadContext.set(threadPool.getThreadContext()); List components = new ArrayList<>(); securityContext.set(new SecurityContext(settings, threadPool.getThreadContext())); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java b/plugin/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java index d7a3ac1781e..72df14cc19f 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java @@ -57,7 +57,7 @@ public class SecurityLifecycleService extends AbstractComponent implements Clust private final IndexLifecycleManager securityIndex; public SecurityLifecycleService(Settings settings, ClusterService clusterService, - ThreadPool threadPool, InternalClient client, + ThreadPool threadPool, InternalSecurityClient client, @Nullable IndexAuditTrail indexAuditTrail) { super(settings); this.settings = settings; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java b/plugin/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java index e89031f7cff..70403c5e258 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java @@ -48,6 +48,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportMessage; import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.security.InternalClient; +import org.elasticsearch.xpack.security.InternalSecurityClient; import org.elasticsearch.xpack.security.audit.AuditLevel; import org.elasticsearch.xpack.security.audit.AuditTrail; import org.elasticsearch.xpack.security.authc.AuthenticationToken; @@ -177,7 +178,7 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail, Cl return NAME; } - public IndexAuditTrail(Settings settings, InternalClient client, ThreadPool threadPool, ClusterService clusterService) { + public IndexAuditTrail(Settings settings, InternalSecurityClient client, ThreadPool threadPool, ClusterService clusterService) { super(settings); this.threadPool = threadPool; this.clusterService = clusterService; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java b/plugin/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java index 455e5491ccc..2f9145da1f6 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool.Names; import org.elasticsearch.xpack.security.InternalClient; +import org.elasticsearch.xpack.security.InternalSecurityClient; import org.elasticsearch.xpack.security.SecurityLifecycleService; import java.time.Instant; @@ -30,12 +31,12 @@ import static org.elasticsearch.action.support.TransportActions.isShardNotAvaila */ final class ExpiredTokenRemover extends AbstractRunnable { - private final InternalClient client; + private final InternalSecurityClient client; private final AtomicBoolean inProgress = new AtomicBoolean(false); private final Logger logger; private final TimeValue timeout; - ExpiredTokenRemover(Settings settings, InternalClient internalClient) { + ExpiredTokenRemover(Settings settings, InternalSecurityClient internalClient) { this.client = internalClient; this.logger = Loggers.getLogger(getClass(), settings); this.timeout = TokenService.DELETE_TIMEOUT.get(settings); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/plugin/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 3228574c169..ee2943a8bfd 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -50,6 +50,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.XPackSettings; import org.elasticsearch.xpack.security.InternalClient; +import org.elasticsearch.xpack.security.InternalSecurityClient; import org.elasticsearch.xpack.security.SecurityLifecycleService; import javax.crypto.Cipher; @@ -132,7 +133,7 @@ public final class TokenService extends AbstractComponent { private final Clock clock; private final TimeValue expirationDelay; private final TimeValue deleteInterval; - private final InternalClient internalClient; + private final InternalSecurityClient internalClient; private final SecurityLifecycleService lifecycleService; private final ExpiredTokenRemover expiredTokenRemover; private final boolean enabled; @@ -148,7 +149,7 @@ public final class TokenService extends AbstractComponent { * @param clock the clock that will be used for comparing timestamps * @param internalClient the client to use when checking for revocations */ - public TokenService(Settings settings, Clock clock, InternalClient internalClient, + public TokenService(Settings settings, Clock clock, InternalSecurityClient internalClient, SecurityLifecycleService lifecycleService, ClusterService clusterService) throws GeneralSecurityException { super(settings); byte[] saltArr = new byte[SALT_BYTES]; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java b/plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java index 0269bd4a13b..715cfbcec73 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.security.InternalClient; +import org.elasticsearch.xpack.security.InternalSecurityClient; import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.action.realm.ClearRealmCacheRequest; import org.elasticsearch.xpack.security.action.realm.ClearRealmCacheResponse; @@ -73,12 +74,12 @@ public class NativeUsersStore extends AbstractComponent { private final Hasher hasher = Hasher.BCRYPT; - private final InternalClient client; + private final InternalSecurityClient client; private final boolean isTribeNode; private volatile SecurityLifecycleService securityLifecycleService; - public NativeUsersStore(Settings settings, InternalClient client, SecurityLifecycleService securityLifecycleService) { + public NativeUsersStore(Settings settings, InternalSecurityClient client, SecurityLifecycleService securityLifecycleService) { super(settings); this.client = client; this.isTribeNode = XPackPlugin.isTribeNode(settings); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java b/plugin/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java index 3814d1d5f68..36fdc3e7816 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java @@ -38,6 +38,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.security.InternalClient; +import org.elasticsearch.xpack.security.InternalSecurityClient; import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.action.rolemapping.DeleteRoleMappingRequest; import org.elasticsearch.xpack.security.action.rolemapping.PutRoleMappingRequest; @@ -70,12 +71,12 @@ public class NativeRoleMappingStore extends AbstractComponent implements UserRol private static final String SECURITY_GENERIC_TYPE = "doc"; - private final InternalClient client; + private final InternalSecurityClient client; private final boolean isTribeNode; private final SecurityLifecycleService securityLifecycleService; private final List realmsToRefresh = new CopyOnWriteArrayList<>(); - public NativeRoleMappingStore(Settings settings, InternalClient client, SecurityLifecycleService securityLifecycleService) { + public NativeRoleMappingStore(Settings settings, InternalSecurityClient client, SecurityLifecycleService securityLifecycleService) { super(settings); this.client = client; this.isTribeNode = XPackPlugin.isTribeNode(settings); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/plugin/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java index ffde645f299..52b650e952f 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java @@ -68,6 +68,7 @@ import org.elasticsearch.xpack.security.support.Automatons; import org.elasticsearch.xpack.security.user.AnonymousUser; import org.elasticsearch.xpack.security.user.SystemUser; import org.elasticsearch.xpack.security.user.User; +import org.elasticsearch.xpack.security.user.XPackSecurityUser; import org.elasticsearch.xpack.security.user.XPackUser; import static org.elasticsearch.xpack.security.Security.setting; @@ -290,7 +291,6 @@ public class AuthorizationService extends AbstractComponent { throw denial(authentication, action, request); } else if (indicesAccessControl.getIndexPermissions(SecurityLifecycleService.SECURITY_INDEX_NAME) != null && indicesAccessControl.getIndexPermissions(SecurityLifecycleService.SECURITY_INDEX_NAME).isGranted() - && XPackUser.is(authentication.getUser()) == false && MONITOR_INDEX_PREDICATE.test(action) == false && isSuperuser(authentication.getUser()) == false) { // only the XPackUser is allowed to work with this index, but we should allow indices monitoring actions through for debugging @@ -392,7 +392,11 @@ public class AuthorizationService extends AbstractComponent { " roles"); } if (XPackUser.is(user)) { - assert XPackUser.INSTANCE.roles().length == 1 && ReservedRolesStore.SUPERUSER_ROLE.name().equals(XPackUser.INSTANCE.roles()[0]); + assert XPackUser.INSTANCE.roles().length == 1; + roleActionListener.onResponse(XPackUser.ROLE); + return; + } + if (XPackSecurityUser.is(user)) { roleActionListener.onResponse(ReservedRolesStore.SUPERUSER_ROLE); return; } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java index 3bec9ec7667..b8109b2cbef 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java @@ -38,6 +38,7 @@ import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.security.InternalClient; +import org.elasticsearch.xpack.security.InternalSecurityClient; import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.action.role.ClearRolesCacheRequest; import org.elasticsearch.xpack.security.action.role.ClearRolesCacheResponse; @@ -79,14 +80,14 @@ public class NativeRolesStore extends AbstractComponent { TimeValue.timeValueMinutes(20), Property.NodeScope, Property.Deprecated); private static final String ROLE_DOC_TYPE = "doc"; - private final InternalClient client; + private final InternalSecurityClient client; private final XPackLicenseState licenseState; private final boolean isTribeNode; private SecurityClient securityClient; private final SecurityLifecycleService securityLifecycleService; - public NativeRolesStore(Settings settings, InternalClient client, XPackLicenseState licenseState, + public NativeRolesStore(Settings settings, InternalSecurityClient client, XPackLicenseState licenseState, SecurityLifecycleService securityLifecycleService) { super(settings); this.client = client; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/ReservedRolesStore.java b/plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/ReservedRolesStore.java index 585176f6c62..6967394bf85 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/ReservedRolesStore.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/ReservedRolesStore.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.security.authz.permission.Role; import org.elasticsearch.xpack.security.support.MetadataUtils; import org.elasticsearch.xpack.security.user.KibanaUser; import org.elasticsearch.xpack.security.user.SystemUser; +import org.elasticsearch.xpack.security.user.XPackUser; import org.elasticsearch.xpack.watcher.execution.TriggeredWatchStore; import org.elasticsearch.xpack.watcher.history.HistoryStore; import org.elasticsearch.xpack.watcher.watch.Watch; @@ -126,7 +127,7 @@ public class ReservedRolesStore { } public static boolean isReserved(String role) { - return RESERVED_ROLES.containsKey(role) || SystemUser.ROLE_NAME.equals(role); + return RESERVED_ROLES.containsKey(role) || SystemUser.ROLE_NAME.equals(role) || XPackUser.ROLE_NAME.equals(role); } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/support/IndexLifecycleManager.java b/plugin/src/main/java/org/elasticsearch/xpack/security/support/IndexLifecycleManager.java index 8cb5282a140..eee1ae4d700 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/support/IndexLifecycleManager.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/support/IndexLifecycleManager.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.xpack.security.InternalClient; +import org.elasticsearch.xpack.security.InternalSecurityClient; import org.elasticsearch.xpack.template.TemplateUtils; import org.elasticsearch.xpack.upgrade.IndexUpgradeCheck; @@ -58,7 +59,7 @@ public class IndexLifecycleManager extends AbstractComponent { private final String indexName; private final String templateName; - private final InternalClient client; + private final InternalSecurityClient client; private final List> indexHealthChangeListeners = new CopyOnWriteArrayList<>(); @@ -70,7 +71,7 @@ public class IndexLifecycleManager extends AbstractComponent { private volatile boolean mappingIsUpToDate; private volatile Version mappingVersion; - public IndexLifecycleManager(Settings settings, InternalClient client, String indexName, String templateName) { + public IndexLifecycleManager(Settings settings, InternalSecurityClient client, String indexName, String templateName) { super(settings); this.client = client; this.indexName = indexName; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/user/User.java b/plugin/src/main/java/org/elasticsearch/xpack/security/user/User.java index f67464c1910..1fecc61aa75 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/user/User.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/user/User.java @@ -184,6 +184,8 @@ public class User implements ToXContentObject { return SystemUser.INSTANCE; } else if (XPackUser.is(username)) { return XPackUser.INSTANCE; + } else if (XPackSecurityUser.is(username)) { + return XPackSecurityUser.INSTANCE; } throw new IllegalStateException("user [" + username + "] is not an internal user"); } @@ -214,6 +216,9 @@ public class User implements ToXContentObject { } else if (XPackUser.is(user)) { output.writeBoolean(true); output.writeString(XPackUser.NAME); + } else if (XPackSecurityUser.is(user)) { + output.writeBoolean(true); + output.writeString(XPackSecurityUser.NAME); } else { if (user.authenticatedUser == null) { // no backcompat necessary, since there is no inner user diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/user/XPackSecurityUser.java b/plugin/src/main/java/org/elasticsearch/xpack/security/user/XPackSecurityUser.java new file mode 100644 index 00000000000..b4c0cc790e9 --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/user/XPackSecurityUser.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.security.user; + +/** + * internal user that manages xpack security. Has all cluster/indices permissions. + */ +public class XPackSecurityUser extends User { + + public static final String NAME = "_xpack_security"; + public static final XPackSecurityUser INSTANCE = new XPackSecurityUser(); + private static final String ROLE_NAME = "superuser"; + + private XPackSecurityUser() { + super(NAME, ROLE_NAME); + } + + @Override + public boolean equals(Object o) { + return INSTANCE == o; + } + + @Override + public int hashCode() { + return System.identityHashCode(this); + } + + public static boolean is(User user) { + return INSTANCE.equals(user); + } + + public static boolean is(String principal) { + return NAME.equals(principal); + } +} diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/user/XPackUser.java b/plugin/src/main/java/org/elasticsearch/xpack/security/user/XPackUser.java index 2f95cd9998d..56b5ecce224 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/user/XPackUser.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/user/XPackUser.java @@ -5,13 +5,22 @@ */ package org.elasticsearch.xpack.security.user; +import org.elasticsearch.xpack.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.security.authz.permission.Role; +import org.elasticsearch.xpack.security.support.MetadataUtils; + /** - * XPack internal user that manages xpack. Has all cluster/indices permissions for x-pack to operate. + * XPack internal user that manages xpack. Has all cluster/indices permissions for x-pack to operate excluding security permissions. */ public class XPackUser extends User { public static final String NAME = "_xpack"; - private static final String ROLE_NAME = "superuser"; + public static final String ROLE_NAME = NAME; + public static final Role ROLE = Role.builder(new RoleDescriptor(ROLE_NAME, new String[] { "all" }, + new RoleDescriptor.IndicesPrivileges[] { + RoleDescriptor.IndicesPrivileges.builder().indices("/@&~(\\.security*)/").privileges("all").build()}, + new String[] { "*" }, + MetadataUtils.DEFAULT_RESERVED_METADATA), null).build(); public static final XPackUser INSTANCE = new XPackUser(); private XPackUser() { diff --git a/plugin/src/main/java/org/elasticsearch/xpack/upgrade/Upgrade.java b/plugin/src/main/java/org/elasticsearch/xpack/upgrade/Upgrade.java index 8868047d107..627082f8bc5 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/upgrade/Upgrade.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/upgrade/Upgrade.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.upgrade; import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -24,6 +25,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xpack.security.InternalClient; +import org.elasticsearch.xpack.security.InternalSecurityClient; import org.elasticsearch.xpack.upgrade.actions.IndexUpgradeAction; import org.elasticsearch.xpack.upgrade.actions.IndexUpgradeInfoAction; import org.elasticsearch.xpack.upgrade.rest.RestIndexUpgradeAction; @@ -53,12 +55,13 @@ public class Upgrade implements ActionPlugin { this.upgradeCheckFactories = new ArrayList<>(); } - public Collection createComponents(InternalClient internalClient, ClusterService clusterService, ThreadPool threadPool, + public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry) { + final InternalSecurityClient internalSecurityClient = new InternalSecurityClient(settings, threadPool, client); List upgradeChecks = new ArrayList<>(upgradeCheckFactories.size()); for (BiFunction checkFactory : upgradeCheckFactories) { - upgradeChecks.add(checkFactory.apply(internalClient, clusterService)); + upgradeChecks.add(checkFactory.apply(internalSecurityClient, clusterService)); } return Collections.singletonList(new IndexUpgradeService(settings, Collections.unmodifiableList(upgradeChecks))); } diff --git a/plugin/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/plugin/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java index 20ab893812d..3d3e378d5eb 100644 --- a/plugin/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java +++ b/plugin/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java @@ -35,6 +35,7 @@ import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.XPackSettings; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.security.InternalClient; +import org.elasticsearch.xpack.security.InternalSecurityClient; import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.client.SecurityClient; import org.junit.AfterClass; @@ -438,6 +439,11 @@ public abstract class SecurityIntegTestCase extends ESIntegTestCase { return internalCluster().getInstance(InternalClient.class); } + protected InternalSecurityClient internalSecurityClient() { + Client client = client(); + return new InternalSecurityClient(client.settings(), client.threadPool(), client); + } + protected SecurityClient securityClient() { return securityClient(client()); } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/SecurityLifecycleServiceTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/SecurityLifecycleServiceTests.java index 8e121c861b6..242d5134331 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/SecurityLifecycleServiceTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/SecurityLifecycleServiceTests.java @@ -64,7 +64,7 @@ public class SecurityLifecycleServiceTests extends ESTestCase { threadPool = new TestThreadPool("security template service tests"); transportClient = new MockTransportClient(Settings.EMPTY); - class IClient extends InternalClient { + class IClient extends InternalSecurityClient { IClient(Client transportClient) { super(Settings.EMPTY, null, transportClient); } @@ -79,7 +79,7 @@ public class SecurityLifecycleServiceTests extends ESTestCase { } } - InternalClient client = new IClient(transportClient); + InternalSecurityClient client = new IClient(transportClient); securityLifecycleService = new SecurityLifecycleService(Settings.EMPTY, clusterService, threadPool, client, mock(IndexAuditTrail.class)); listeners = new CopyOnWriteArrayList<>(); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java index d0c4095a3dd..768a4e3e346 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java @@ -77,9 +77,9 @@ public class SecurityTests extends ESTestCase { allowedSettings.addAll(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); ClusterSettings clusterSettings = new ClusterSettings(settings, allowedSettings); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); - InternalClient client = new InternalClient(Settings.EMPTY, threadPool, mock(Client.class)); when(threadPool.relativeTimeInMillis()).thenReturn(1L); - return security.createComponents(client, threadPool, clusterService, mock(ResourceWatcherService.class), Arrays.asList(extensions)); + return security.createComponents(mock(Client.class), threadPool, clusterService, mock(ResourceWatcherService.class), + Arrays.asList(extensions)); } private T findComponent(Class type, Collection components) { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/AuditTrailTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/AuditTrailTests.java index 7b57b1825c6..2227c17c58e 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/AuditTrailTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/AuditTrailTests.java @@ -141,7 +141,7 @@ public class AuditTrailTests extends SecurityIntegTestCase { return eventsRef.get(); } private Collection> getAuditEvents() throws Exception { - final InternalClient client = internalClient(); + final InternalClient client = internalSecurityClient(); DateTime now = new DateTime(DateTimeZone.UTC); String indexName = IndexNameResolver.resolve(IndexAuditTrail.INDEX_NAME_PREFIX, now, IndexNameResolver.Rollover.DAILY); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailMutedTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailMutedTests.java index fd685b33db0..7d2eb216249 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailMutedTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailMutedTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.MockTransportClient; import org.elasticsearch.transport.TransportMessage; import org.elasticsearch.xpack.security.InternalClient; +import org.elasticsearch.xpack.security.InternalSecurityClient; import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail.State; import org.elasticsearch.xpack.security.authc.AuthenticationToken; import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; @@ -42,7 +43,7 @@ import static org.mockito.Mockito.when; public class IndexAuditTrailMutedTests extends ESTestCase { - private InternalClient client; + private InternalSecurityClient client; private TransportClient transportClient; private ThreadPool threadPool; private ClusterService clusterService; @@ -61,7 +62,7 @@ public class IndexAuditTrailMutedTests extends ESTestCase { threadPool = new TestThreadPool("index audit trail tests"); transportClient = new MockTransportClient(Settings.EMPTY); clientCalled = new AtomicBoolean(false); - class IClient extends InternalClient { + class IClient extends InternalSecurityClient { IClient(Client transportClient){ super(Settings.EMPTY, threadPool, transportClient); } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java index cbaf1f5aab6..0943f19ef1e 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java @@ -295,7 +295,7 @@ public class IndexAuditTrailTests extends SecurityIntegTestCase { when(nodes.isLocalNodeElectedMaster()).thenReturn(true); threadPool = new TestThreadPool("index audit trail tests"); enqueuedMessage = new SetOnce<>(); - auditor = new IndexAuditTrail(settings, internalClient(), threadPool, clusterService) { + auditor = new IndexAuditTrail(settings, internalSecurityClient(), threadPool, clusterService) { @Override void enqueue(Message message, String type) { enqueuedMessage.set(message); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailUpdateMappingTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailUpdateMappingTests.java index 9bad1d5c10b..9e7a98ce0e1 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailUpdateMappingTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailUpdateMappingTests.java @@ -50,7 +50,7 @@ public class IndexAuditTrailUpdateMappingTests extends SecurityIntegTestCase { when(localNode.getHostAddress()).thenReturn(buildNewFakeTransportAddress().toString()); ClusterService clusterService = mock(ClusterService.class); when(clusterService.localNode()).thenReturn(localNode); - auditor = new IndexAuditTrail(settings, internalClient(), threadPool, clusterService); + auditor = new IndexAuditTrail(settings, internalSecurityClient(), threadPool, clusterService); // before starting we add an event auditor.authenticationFailed(new FakeRestRequest()); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index 96592d47b55..0aade415557 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -48,6 +48,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportMessage; import org.elasticsearch.xpack.XPackSettings; import org.elasticsearch.xpack.security.InternalClient; +import org.elasticsearch.xpack.security.InternalSecurityClient; import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.audit.AuditTrailService; import org.elasticsearch.xpack.security.authc.Authentication.RealmRef; @@ -135,7 +136,7 @@ public class AuthenticationServiceTests extends ESTestCase { threadPool = new ThreadPool(settings, new FixedExecutorBuilder(settings, TokenService.THREAD_POOL_NAME, 1, 1000, "xpack.security.authc.token.thread_pool")); threadContext = threadPool.getThreadContext(); - InternalClient internalClient = new InternalClient(Settings.EMPTY, threadPool, client); + InternalSecurityClient internalClient = new InternalSecurityClient(Settings.EMPTY, threadPool, client); lifecycleService = mock(SecurityLifecycleService.class); ClusterService clusterService = new ClusterService(settings, new ClusterSettings(settings, ClusterSettings .BUILT_IN_CLUSTER_SETTINGS), threadPool, Collections.emptyMap()); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java index e5df63c1429..8706c9dd48b 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java @@ -56,7 +56,7 @@ public class TokenAuthIntegTests extends SecurityIntegTestCase { } public void testTokenServiceBootstrapOnNodeJoin() throws Exception { - final Client client = internalClient(); + final Client client = internalSecurityClient(); SecurityClient securityClient = new SecurityClient(client); CreateTokenResponse response = securityClient.prepareCreateToken() .setGrantType("password") @@ -84,7 +84,7 @@ public class TokenAuthIntegTests extends SecurityIntegTestCase { public void testTokenServiceCanRotateKeys() throws Exception { - final Client client = internalClient(); + final Client client = internalSecurityClient(); SecurityClient securityClient = new SecurityClient(client); CreateTokenResponse response = securityClient.prepareCreateToken() .setGrantType("password") @@ -116,7 +116,7 @@ public class TokenAuthIntegTests extends SecurityIntegTestCase { } public void testExpiredTokensDeletedAfterExpiration() throws Exception { - final Client client = internalClient(); + final Client client = internalSecurityClient(); SecurityClient securityClient = new SecurityClient(client); CreateTokenResponse response = securityClient.prepareCreateToken() .setGrantType("password") diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index 31e8e1d924e..a973e3e1492 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.XPackSettings; import org.elasticsearch.xpack.security.InternalClient; +import org.elasticsearch.xpack.security.InternalSecurityClient; import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.security.authc.TokenService.BytesKey; @@ -49,7 +50,7 @@ import static org.mockito.Mockito.when; public class TokenServiceTests extends ESTestCase { - private InternalClient internalClient; + private InternalSecurityClient internalClient; private static ThreadPool threadPool; private static final Settings settings = Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "TokenServiceTests") .put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), true).build(); @@ -63,7 +64,7 @@ public class TokenServiceTests extends ESTestCase { @Before public void setupClient() throws GeneralSecurityException { client = mock(Client.class); - internalClient = new InternalClient(settings, threadPool, client); + internalClient = new InternalSecurityClient(settings, threadPool, client); lifecycleService = mock(SecurityLifecycleService.class); when(lifecycleService.isSecurityIndexWriteable()).thenReturn(true); doAnswer(invocationOnMock -> { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java index b75c314a23b..c92e2640b36 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.security.InternalClient; +import org.elasticsearch.xpack.security.InternalSecurityClient; import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authc.AuthenticationResult; import org.elasticsearch.xpack.security.authc.support.Hasher; @@ -54,12 +55,12 @@ public class NativeUsersStoreTests extends ESTestCase { private static final String PASSWORD_FIELD = User.Fields.PASSWORD.getPreferredName(); private static final String BLANK_PASSWORD = ""; - private InternalClient internalClient; + private InternalSecurityClient internalClient; private final List>> requests = new CopyOnWriteArrayList<>(); @Before public void setupMocks() { - internalClient = new InternalClient(Settings.EMPTY, null, null) { + internalClient = new InternalSecurityClient(Settings.EMPTY, null, null) { @Override protected < diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeUserRoleMapperTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeUserRoleMapperTests.java index 39f5146f8ca..819b4f6ab1c 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeUserRoleMapperTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeUserRoleMapperTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.env.Environment; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.security.InternalClient; +import org.elasticsearch.xpack.security.InternalSecurityClient; import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authc.RealmConfig; import org.elasticsearch.xpack.security.authc.support.UserRoleMapper; @@ -53,7 +54,7 @@ public class NativeUserRoleMapperTests extends ESTestCase { Collections.singletonList(FieldPredicate.create("cn=mutants,ou=groups,ou=dept_h,o=forces,dc=gc,dc=ca"))), Arrays.asList("mutants"), Collections.emptyMap(), false); - final InternalClient client = mock(InternalClient.class); + final InternalSecurityClient client = mock(InternalSecurityClient.class); final SecurityLifecycleService lifecycleService = mock(SecurityLifecycleService.class); when(lifecycleService.isSecurityIndexAvailable()).thenReturn(true); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java index f40f215e0a3..71aad327267 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java @@ -715,7 +715,7 @@ public class AuthorizationServiceTests extends ESTestCase { } } - public void testXPackUserAndSuperusersCanExecuteOperationAgainstSecurityIndex() { + public void testSuperusersCanExecuteOperationAgainstSecurityIndex() { final User superuser = new User("custom_admin", ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName()); roleMap.put(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName(), ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR); ClusterState state = mock(ClusterState.class); @@ -726,37 +726,35 @@ public class AuthorizationServiceTests extends ESTestCase { .numberOfShards(1).numberOfReplicas(0).build(), true) .build()); - for (User user : Arrays.asList(XPackUser.INSTANCE, superuser)) { - List> requests = new ArrayList<>(); - requests.add(new Tuple<>(DeleteAction.NAME, new DeleteRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(BulkAction.NAME + "[s]", - createBulkShardRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, DeleteRequest::new))); - requests.add(new Tuple<>(UpdateAction.NAME, new UpdateRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(IndexAction.NAME, new IndexRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(BulkAction.NAME + "[s]", - createBulkShardRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, IndexRequest::new))); - requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(SecurityLifecycleService.SECURITY_INDEX_NAME))); - requests.add(new Tuple<>(TermVectorsAction.NAME, - new TermVectorsRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(GetAction.NAME, new GetRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(TermVectorsAction.NAME, - new TermVectorsRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(IndicesAliasesAction.NAME, new IndicesAliasesRequest() - .addAliasAction(AliasActions.add().alias("security_alias").index(SecurityLifecycleService.SECURITY_INDEX_NAME)))); - requests.add(new Tuple<>(ClusterHealthAction.NAME, new ClusterHealthRequest(SecurityLifecycleService.SECURITY_INDEX_NAME))); - requests.add(new Tuple<>(ClusterHealthAction.NAME, - new ClusterHealthRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "foo", "bar"))); + List> requests = new ArrayList<>(); + requests.add(new Tuple<>(DeleteAction.NAME, new DeleteRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id"))); + requests.add(new Tuple<>(BulkAction.NAME + "[s]", + createBulkShardRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, DeleteRequest::new))); + requests.add(new Tuple<>(UpdateAction.NAME, new UpdateRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id"))); + requests.add(new Tuple<>(IndexAction.NAME, new IndexRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id"))); + requests.add(new Tuple<>(BulkAction.NAME + "[s]", + createBulkShardRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, IndexRequest::new))); + requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(SecurityLifecycleService.SECURITY_INDEX_NAME))); + requests.add(new Tuple<>(TermVectorsAction.NAME, + new TermVectorsRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id"))); + requests.add(new Tuple<>(GetAction.NAME, new GetRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id"))); + requests.add(new Tuple<>(TermVectorsAction.NAME, + new TermVectorsRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id"))); + requests.add(new Tuple<>(IndicesAliasesAction.NAME, new IndicesAliasesRequest() + .addAliasAction(AliasActions.add().alias("security_alias").index(SecurityLifecycleService.SECURITY_INDEX_NAME)))); + requests.add(new Tuple<>(ClusterHealthAction.NAME, new ClusterHealthRequest(SecurityLifecycleService.SECURITY_INDEX_NAME))); + requests.add(new Tuple<>(ClusterHealthAction.NAME, + new ClusterHealthRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "foo", "bar"))); - for (Tuple requestTuple : requests) { - String action = requestTuple.v1(); - TransportRequest request = requestTuple.v2(); - authorize(createAuthentication(user), action, request); - verify(auditTrail).accessGranted(user, action, request); - } + for (Tuple requestTuple : requests) { + String action = requestTuple.v1(); + TransportRequest request = requestTuple.v2(); + authorize(createAuthentication(superuser), action, request); + verify(auditTrail).accessGranted(superuser, action, request); } } - public void testXPackUserAndSuperusersCanExecuteOperationAgainstSecurityIndexWithWildcard() { + public void testSuperusersCanExecuteOperationAgainstSecurityIndexWithWildcard() { final User superuser = new User("custom_admin", ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName()); roleMap.put(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName(), ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR); ClusterState state = mock(ClusterState.class); @@ -769,11 +767,6 @@ public class AuthorizationServiceTests extends ESTestCase { String action = SearchAction.NAME; SearchRequest request = new SearchRequest("_all"); - authorize(createAuthentication(XPackUser.INSTANCE), action, request); - verify(auditTrail).accessGranted(XPackUser.INSTANCE, action, request); - assertThat(request.indices(), arrayContaining(".security")); - - request = new SearchRequest("_all"); authorize(createAuthentication(superuser), action, request); verify(auditTrail).accessGranted(superuser, action, request); assertThat(request.indices(), arrayContaining(".security")); @@ -1073,7 +1066,7 @@ public class AuthorizationServiceTests extends ESTestCase { PlainActionFuture rolesFuture = new PlainActionFuture<>(); authorizationService.roles(XPackUser.INSTANCE, rolesFuture); final Role roles = rolesFuture.actionGet(); - assertThat(roles, equalTo(ReservedRolesStore.SUPERUSER_ROLE)); + assertThat(roles, equalTo(XPackUser.ROLE)); verifyZeroInteractions(rolesStore); } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java index b703c1cfe95..8ad435db957 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java @@ -69,6 +69,7 @@ import org.elasticsearch.xpack.security.authz.store.ReservedRolesStore; import org.elasticsearch.xpack.security.test.SecurityTestUtils; import org.elasticsearch.xpack.security.user.AnonymousUser; import org.elasticsearch.xpack.security.user.User; +import org.elasticsearch.xpack.security.user.XPackSecurityUser; import org.elasticsearch.xpack.security.user.XPackUser; import org.junit.Before; @@ -1191,22 +1192,29 @@ public class IndicesAndAliasesResolverTests extends ESTestCase { } } - public void testXPackUserHasAccessToSecurityIndex() { + public void testXPackSecurityUserHasAccessToSecurityIndex() { SearchRequest request = new SearchRequest(); { - final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackUser.INSTANCE, SearchAction.NAME); + final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, SearchAction.NAME); List indices = resolveIndices(request, authorizedIndices).getLocal(); assertThat(indices, hasItem(SecurityLifecycleService.SECURITY_INDEX_NAME)); } { IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest(); aliasesRequest.addAliasAction(AliasActions.add().alias("security_alias").index(SecurityLifecycleService.SECURITY_INDEX_NAME)); - final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackUser.INSTANCE, IndicesAliasesAction.NAME); + final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, IndicesAliasesAction.NAME); List indices = resolveIndices(aliasesRequest, authorizedIndices).getLocal(); assertThat(indices, hasItem(SecurityLifecycleService.SECURITY_INDEX_NAME)); } } + public void testXPackUserDoesNotHaveAccessToSecurityIndex() { + SearchRequest request = new SearchRequest(); + final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackUser.INSTANCE, SearchAction.NAME); + List indices = resolveIndices(request, authorizedIndices).getLocal(); + assertThat(indices, not(hasItem(SecurityLifecycleService.SECURITY_INDEX_NAME))); + } + public void testNonXPackUserAccessingSecurityIndex() { User allAccessUser = new User("all_access", "all_access"); roleMap.put("all_access", new RoleDescriptor("all_access", new String[] { "all" }, diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java index f51abbb4c54..e78a2301ff9 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java @@ -38,6 +38,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.security.InternalClient; +import org.elasticsearch.xpack.security.InternalSecurityClient; import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.action.role.PutRoleRequest; import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail; @@ -184,7 +185,7 @@ public class NativeRolesStoreTests extends ESTestCase { } public void testPutOfRoleWithFlsDlsUnlicensed() throws IOException { - final InternalClient internalClient = mock(InternalClient.class); + final InternalSecurityClient internalClient = mock(InternalSecurityClient.class); final ClusterService clusterService = mock(ClusterService.class); final XPackLicenseState licenseState = mock(XPackLicenseState.class); final AtomicBoolean methodCalled = new AtomicBoolean(false); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/authz/store/ReservedRolesStoreTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/authz/store/ReservedRolesStoreTests.java index d1741ca2b0e..438d3498940 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/authz/store/ReservedRolesStoreTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/authz/store/ReservedRolesStoreTests.java @@ -80,6 +80,7 @@ import org.elasticsearch.xpack.security.authz.accesscontrol.IndicesAccessControl import org.elasticsearch.xpack.security.authz.permission.FieldPermissionsCache; import org.elasticsearch.xpack.security.authz.permission.Role; import org.elasticsearch.xpack.security.user.SystemUser; +import org.elasticsearch.xpack.security.user.XPackUser; import org.elasticsearch.xpack.watcher.execution.TriggeredWatchStore; import org.elasticsearch.xpack.watcher.history.HistoryStore; import org.elasticsearch.xpack.watcher.transport.actions.ack.AckWatchAction; @@ -123,6 +124,7 @@ public class ReservedRolesStoreTests extends ESTestCase { assertThat(ReservedRolesStore.isReserved("watcher_user"), is(true)); assertThat(ReservedRolesStore.isReserved("watcher_admin"), is(true)); assertThat(ReservedRolesStore.isReserved("kibana_dashboard_only_user"), is(true)); + assertThat(ReservedRolesStore.isReserved(XPackUser.ROLE_NAME), is(true)); } public void testIngestAdminRole() { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/support/IndexLifecycleManagerTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/support/IndexLifecycleManagerTests.java index dbb4d74b20c..731325dfd42 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/support/IndexLifecycleManagerTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/support/IndexLifecycleManagerTests.java @@ -45,6 +45,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.security.InternalClient; +import org.elasticsearch.xpack.security.InternalSecurityClient; import org.elasticsearch.xpack.security.test.SecurityTestUtils; import org.elasticsearch.xpack.template.TemplateUtils; import org.hamcrest.Matchers; @@ -71,7 +72,7 @@ public class IndexLifecycleManagerTests extends ESTestCase { when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); actions = new LinkedHashMap<>(); - final InternalClient client = new InternalClient(Settings.EMPTY, threadPool, mockClient) { + final InternalSecurityClient client = new InternalSecurityClient(Settings.EMPTY, threadPool, mockClient) { @Override protected