From a289fbd168e5a7c39798c1c1e1183bbc86954a5b Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 2 Jun 2016 11:50:21 -0600 Subject: [PATCH 01/39] Add a tool to migrate users/roles from file to native realm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds the `bin/shield/migrate` tool that allows migrating users and roles from the files to the native (API-based) store. It looks like this: ``` λ bin/shield/migrate native -U http://localhost:9200 -u test_user -p changeme -n lee,foo -r role1,role2,role3,role4,foo starting migration of users and roles... importing users from [/home/hinmanm/scratch/elasticsearch-2.4.0-SNAPSHOT/config/shield/users]... found existing users: [test_user, joe3, joe2] migrating user [lee] {"user":{"created":true}} no user [foo] found, skipping importing roles from [/home/hinmanm/scratch/elasticsearch-2.4.0-SNAPSHOT/config/shield/roles.yml]... found existing roles: [marvel_user, role_query_fields, admin_role, role3, admin, remote_marvel_agent, power_user, role_new_format_name_array, role_run_as, logstash, role_fields, role_run_as1, role_new_format, kibana4_server, user, transport_client, role1.ab, role_query] migrating role [role1] {"role":{"created":true}} migrating role [role2] {"role":{"created":true}} role [role3] already exists, skipping migrating role [role4] failed to migrate role [role4] with body: {"indices":[{"names":["idx2"]},{"names":["idx2"]},{"names":["idx1"]}]} java.io.IOException: {"error":{"root_cause":[{"type":"parse_exception","reason":"failed to parse indices privileges for role [role4]. missing required [privileges] field"}],"type":"parse_exception","reason":"failed to parse indices privileges for role [role4]. missing required [privileges] field"},"status":400} at org.elasticsearch.shield.authc.esusers.tool.ESNativeRealmMigrateTool$MigrateUserOrRoles.postURL(ESNativeRealmMigrateTool.java:206) at org.elasticsearch.shield.authc.esusers.tool.ESNativeRealmMigrateTool$MigrateUserOrRoles.importRoles(ESNativeRealmMigrateTool.java:389) at org.elasticsearch.shield.authc.esusers.tool.ESNativeRealmMigrateTool$MigrateUserOrRoles.execute(ESNativeRealmMigrateTool.java:171) at org.elasticsearch.common.cli.CliTool.execute(CliTool.java:153) at org.elasticsearch.shield.authc.esusers.tool.ESNativeRealmMigrateTool.main(ESNativeRealmMigrateTool.java:91) Caused by: java.io.IOException: Server returned HTTP response code: 400 for URL: http://localhost:9200/_shield/role/role4 at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1840) at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1441) at org.elasticsearch.shield.authc.esusers.tool.ESNativeRealmMigrateTool$MigrateUserOrRoles.postURL(ESNativeRealmMigrateTool.java:192) ... 4 more no role [foo] found, skipping users and roles imported. ``` Original commit: elastic/x-pack-elasticsearch@3ce47c0ffd9003df3970ae9ef92c10826ddfdf11 --- .../qa/security-migrate-tests/build.gradle | 32 ++ .../qa/security-migrate-tests/roles.yml | 17 + .../xpack/security/MigrateToolIT.java | 125 ++++++ .../xpack/security/MigrateToolTestCase.java | 177 ++++++++ .../x-pack/security/bin/x-pack/migrate | 102 +++++ .../x-pack/security/bin/x-pack/migrate.bat | 9 + .../action/user/PutUserRequestBuilder.java | 13 + .../esnative/ESNativeRealmMigrateTool.java | 397 ++++++++++++++++++ .../security/authz/store/FileRolesStore.java | 65 ++- .../xpack/security/user/User.java | 1 + .../xpack/security/crypto/migrate.help | 34 ++ .../test/SecurityIntegTestCase.java | 22 +- .../esnative/ESNativeMigrateToolTests.java | 150 +++++++ .../ESNativeRealmMigrateToolTests.java | 50 +++ 14 files changed, 1179 insertions(+), 15 deletions(-) create mode 100644 elasticsearch/qa/security-migrate-tests/build.gradle create mode 100644 elasticsearch/qa/security-migrate-tests/roles.yml create mode 100644 elasticsearch/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java create mode 100644 elasticsearch/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolTestCase.java create mode 100755 elasticsearch/x-pack/security/bin/x-pack/migrate create mode 100644 elasticsearch/x-pack/security/bin/x-pack/migrate.bat create mode 100644 elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateTool.java create mode 100644 elasticsearch/x-pack/security/src/main/resources/org/elasticsearch/xpack/security/crypto/migrate.help create mode 100644 elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java create mode 100644 elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateToolTests.java diff --git a/elasticsearch/qa/security-migrate-tests/build.gradle b/elasticsearch/qa/security-migrate-tests/build.gradle new file mode 100644 index 00000000000..3cc6464e39e --- /dev/null +++ b/elasticsearch/qa/security-migrate-tests/build.gradle @@ -0,0 +1,32 @@ +apply plugin: 'elasticsearch.rest-test' + +dependencies { + testCompile project(path: ':x-plugins:elasticsearch:x-pack', configuration: 'runtime') +} + +integTest { + cluster { + setting 'script.inline', 'true' + plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack') + extraConfigFile 'x-pack/roles.yml', 'roles.yml' + [ + test_admin: 'superuser', + transport_user: 'superuser', + existing: 'superuser', + bob: 'actual_role' + ].each { String user, String role -> + setupCommand 'setupUser#' + user, + 'bin/x-pack/users', 'useradd', user, '-p', 'changeme', '-r', role + } + waitCondition = { node, ant -> + File tmpFile = new File(node.cwd, 'wait.success') + ant.get(src: "http://${node.httpUri()}", + dest: tmpFile.toString(), + username: 'test_admin', + password: 'changeme', + ignoreerrors: true, + retries: 10) + return tmpFile.exists() + } + } +} diff --git a/elasticsearch/qa/security-migrate-tests/roles.yml b/elasticsearch/qa/security-migrate-tests/roles.yml new file mode 100644 index 00000000000..b55b208c3ed --- /dev/null +++ b/elasticsearch/qa/security-migrate-tests/roles.yml @@ -0,0 +1,17 @@ +actual_role: + run_as: [ "joe" ] + cluster: + - monitor + indices: + - names: [ "index1", "index2" ] + privileges: [ "read", "write", "create_index", "indices:admin/refresh" ] + fields: + - foo + - bar + query: + bool: + must_not: + match: + hidden: true + - names: "*" + privileges: [ "read" ] diff --git a/elasticsearch/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java b/elasticsearch/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java new file mode 100644 index 00000000000..5b3b36e5674 --- /dev/null +++ b/elasticsearch/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.security; + +import joptsimple.OptionParser; +import joptsimple.OptionSet; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cli.MockTerminal; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.Requests; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.env.Environment; +import org.elasticsearch.xpack.security.SecurityTemplateService; +import org.elasticsearch.xpack.security.action.role.GetRolesResponse; +import org.elasticsearch.xpack.security.action.user.GetUsersResponse; +import org.elasticsearch.xpack.security.action.user.PutUserResponse; +import org.elasticsearch.xpack.security.authc.esnative.ESNativeRealmMigrateTool; +import org.elasticsearch.xpack.security.authc.support.SecuredString; +import org.elasticsearch.xpack.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.security.client.SecurityClient; +import org.elasticsearch.xpack.security.user.User; + +import java.util.Arrays; +import java.util.Collections; + +import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.hamcrest.Matchers.containsString; + +/** + * Integration tests for the {@code migrate} shell command + */ +public class MigrateToolIT extends MigrateToolTestCase { + + private static String[] args(String command) { + if (!Strings.hasLength(command)) { + return Strings.EMPTY_ARRAY; + } + return command.split("\\s+"); + } + + public void testRunMigrateTool() throws Exception { + String integHome = System.getProperty("tests.config.dir"); + logger.info("--> HOME: {}", integHome); + Settings settings = Settings.builder() + .put("path.home", createTempDir().toAbsolutePath().toString()) + .build(); + // Cluster should already be up + String url = "http://" + getHttpURL(); + logger.info("--> using URL: {}", url); + MockTerminal t = new MockTerminal(); + Client client = getClient(); + SecurityClient c = new SecurityClient(client); + + // Add an existing user so the tool will skip it + PutUserResponse pur = c.preparePutUser("existing", "s3kirt".toCharArray(), "role1", "user").get(); + assertTrue(pur.created()); + + // Wait for the security index to be green + ClusterHealthResponse actionGet = client.admin().cluster() + .health(Requests.clusterHealthRequest(SecurityTemplateService.SECURITY_INDEX_NAME) + .timeout(TimeValue.timeValueSeconds(30)) + .waitForGreenStatus() + .waitForEvents(Priority.LANGUID) + .waitForRelocatingShards(0)) + .actionGet(); + ESNativeRealmMigrateTool.MigrateUserOrRoles muor = new ESNativeRealmMigrateTool.MigrateUserOrRoles(); + OptionParser parser = muor.getParser(); + OptionSet options = parser.parse("-u", "test_admin", "-p", "changeme", "-U", url, "-c", integHome); + muor.execute(t, options, settings.getAsMap()); + + logger.info("--> output:\n{}", t.getOutput()); + + // Check that the migrated user can be retrieved + GetUsersResponse resp = c.prepareGetUsers("bob").get(); + assertTrue("user 'bob' should exist", resp.hasUsers()); + User bob = resp.users()[0]; + assertEquals(bob.principal(), "bob"); + assertArrayEquals(bob.roles(), new String[]{"actual_role"}); + + // Make sure the existing user did not change + resp = c.prepareGetUsers("existing").get(); + assertTrue("user should exist", resp.hasUsers()); + User existing = resp.users()[0]; + assertEquals(existing.principal(), "existing"); + assertArrayEquals(existing.roles(), new String[]{"role1", "user"}); + + // Make sure the "actual_role" made it in and is correct + GetRolesResponse roleResp = c.prepareGetRoles().names("actual_role").get(); + assertTrue("role should exist", roleResp.hasRoles()); + RoleDescriptor rd = roleResp.roles()[0]; + assertNotNull(rd); + assertEquals(rd.getName(), "actual_role"); + assertArrayEquals(rd.getClusterPrivileges(), new String[]{"monitor"}); + assertArrayEquals(rd.getRunAs(), new String[]{"joe"}); + RoleDescriptor.IndicesPrivileges[] ips = rd.getIndicesPrivileges(); + assertEquals(ips.length, 2); + for (RoleDescriptor.IndicesPrivileges ip : ips) { + if (Arrays.equals(ip.getIndices(), new String[]{"index1", "index2"})) { + assertArrayEquals(ip.getPrivileges(), new String[]{"read", "write", "create_index", "indices:admin/refresh"}); + assertArrayEquals(ip.getFields(), new String[]{"foo", "bar"}); + assertNotNull(ip.getQuery()); + assertThat(ip.getQuery().toUtf8(), containsString("{\"bool\":{\"must_not\":{\"match\":{\"hidden\":true}}}}")); + } else { + assertArrayEquals(ip.getIndices(), new String[]{"*"}); + assertArrayEquals(ip.getPrivileges(), new String[]{"read"}); + assertArrayEquals(ip.getFields(), null); + assertNull(ip.getQuery()); + } + } + + // Check that bob can access the things the "actual_role" says he can + String token = basicAuthHeaderValue("bob", new SecuredString("changeme".toCharArray())); + // Create "index1" index and try to search from it as "bob" + client.filterWithHeader(Collections.singletonMap("Authorization", token)).admin().indices().prepareCreate("index1").get(); + SearchResponse searchResp = client.filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("index1").get(); + } +} diff --git a/elasticsearch/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolTestCase.java b/elasticsearch/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolTestCase.java new file mode 100644 index 00000000000..76f57f802ff --- /dev/null +++ b/elasticsearch/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolTestCase.java @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.security; + +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.node.internal.InternalSettingsPreparer; +import org.elasticsearch.xpack.security.Security; +import org.elasticsearch.xpack.XPackPlugin; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.nio.file.Path; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiOfLength; +import static org.hamcrest.Matchers.notNullValue; + +/** + * {@link MigrateToolTestCase} is an abstract base class to run integration + * tests against an external Elasticsearch Cluster. + *

+ * You can define a list of transport addresses from where you can reach your cluster + * by setting "tests.cluster" system property. It defaults to "localhost:9300". + *

+ * All tests can be run from maven using mvn install as maven will start an external cluster first. + *

+ * If you want to debug this module from your IDE, then start an external cluster by yourself + * then run JUnit. If you changed the default port, set "tests.cluster=localhost:PORT" when running + * your test. + */ +@LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose") +public abstract class MigrateToolTestCase extends LuceneTestCase { + + /** + * Key used to eventually switch to using an external cluster and provide its transport addresses + */ + public static final String TESTS_CLUSTER = "tests.cluster"; + + /** + * Key used to eventually switch to using an external cluster and provide its transport addresses + */ + public static final String TESTS_HTTP_CLUSTER = "tests.rest.cluster"; + + /** + * Defaults to localhost:9300 + */ + public static final String TESTS_CLUSTER_DEFAULT = "localhost:9300"; + + protected static final ESLogger logger = ESLoggerFactory.getLogger(MigrateToolTestCase.class.getName()); + + private static final AtomicInteger counter = new AtomicInteger(); + private static Client client; + private static String clusterAddresses; + private static String clusterHttpAddresses; + + private static Client startClient(Path tempDir, TransportAddress... transportAddresses) { + logger.info("--> Starting Elasticsearch Java TransportClient {}, {}", transportAddresses, tempDir); + + Settings clientSettings = Settings.builder() + .put("cluster.name", "qa_migrate_tests_" + counter.getAndIncrement()) + .put("client.transport.ignore_cluster_name", true) + .put("path.home", tempDir) + .put(Security.USER_SETTING.getKey(), "transport_user:changeme") + .put("node.mode", "network") // we require network here! + .build(); + + TransportClient.Builder transportClientBuilder = TransportClient.builder() + .addPlugin(XPackPlugin.class) + .settings(clientSettings); + TransportClient client = transportClientBuilder.build().addTransportAddresses(transportAddresses); + + logger.info("--> Elasticsearch Java TransportClient started"); + + Exception clientException = null; + try { + ClusterHealthResponse health = client.admin().cluster().prepareHealth().get(); + logger.info("--> connected to [{}] cluster which is running [{}] node(s).", + health.getClusterName(), health.getNumberOfNodes()); + } catch (Exception e) { + clientException = e; + } + + assumeNoException("Sounds like your cluster is not running at " + clusterAddresses, clientException); + + return client; + } + + private static Client startClient() throws UnknownHostException { + String[] stringAddresses = clusterAddresses.split(","); + TransportAddress[] transportAddresses = new TransportAddress[stringAddresses.length]; + int i = 0; + for (String stringAddress : stringAddresses) { + int lastColon = stringAddress.lastIndexOf(":"); + if (lastColon == -1) { + throw new IllegalArgumentException("address [" + clusterAddresses + "] not valid"); + } + String ip = stringAddress.substring(0, lastColon); + String port = stringAddress.substring(lastColon + 1); + try { + transportAddresses[i++] = new InetSocketTransportAddress(InetAddress.getByName(ip), Integer.valueOf(port)); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("port is not valid, expected number but was [" + port + "]"); + } + } + return startClient(createTempDir(), transportAddresses); + } + + public static Client getClient() { + if (client == null) { + try { + client = startClient(); + } catch (UnknownHostException e) { + logger.error("could not start the client", e); + } + assertThat(client, notNullValue()); + } + return client; + } + + public static String getHttpURL() { + return clusterHttpAddresses; + } + + @BeforeClass + public static void initializeSettings() throws UnknownHostException { + String port = System.getProperty("integ.http.port"); + clusterAddresses = System.getProperty(TESTS_CLUSTER); + clusterHttpAddresses = System.getProperty(TESTS_HTTP_CLUSTER); + if (clusterAddresses == null || clusterAddresses.isEmpty()) { + throw new UnknownHostException("unable to get a cluster address"); + } + } + + @AfterClass + public static void stopTransportClient() { + if (client != null) { + client.close(); + client = null; + } + } + + @Before + public void defineIndexName() { + doClean(); + } + + @After + public void cleanIndex() { + doClean(); + } + + private void doClean() { + if (client != null) { + try { + client.admin().indices().prepareDelete("_all").get(); + } catch (Exception e) { + // We ignore this cleanup exception + } + } + } +} diff --git a/elasticsearch/x-pack/security/bin/x-pack/migrate b/elasticsearch/x-pack/security/bin/x-pack/migrate new file mode 100755 index 00000000000..a4b35234bb3 --- /dev/null +++ b/elasticsearch/x-pack/security/bin/x-pack/migrate @@ -0,0 +1,102 @@ +#!/bin/bash + +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +SCRIPT="$0" + +# SCRIPT may be an arbitrarily deep series of symlinks. Loop until we have the concrete path. +while [ -h "$SCRIPT" ] ; do + ls=`ls -ld "$SCRIPT"` + # Drop everything prior to -> + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + SCRIPT="$link" + else + SCRIPT=`dirname "$SCRIPT"`/"$link" + fi +done + +# determine elasticsearch home +ES_HOME=`dirname "$SCRIPT"`/../.. + +# make ELASTICSEARCH_HOME absolute +ES_HOME=`cd "$ES_HOME"; pwd` + +# If an include wasn't specified in the environment, then search for one... +if [ "x$ES_INCLUDE" = "x" ]; then + # Locations (in order) to use when searching for an include file. + for include in /usr/share/elasticsearch/elasticsearch.in.sh \ + /usr/local/share/elasticsearch/elasticsearch.in.sh \ + /opt/elasticsearch/elasticsearch.in.sh \ + ~/.elasticsearch.in.sh \ + "`dirname "$0"`"/../elasticsearch.in.sh \ + "$ES_HOME/bin/elasticsearch.in.sh"; do + if [ -r "$include" ]; then + . "$include" + break + fi + done +# ...otherwise, source the specified include. +elif [ -r "$ES_INCLUDE" ]; then + . "$ES_INCLUDE" +fi + +if [ -x "$JAVA_HOME/bin/java" ]; then + JAVA="$JAVA_HOME/bin/java" +else + JAVA=`which java` +fi + +if [ ! -x "$JAVA" ]; then + echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME" + exit 1 +fi + +if [ -z "$ES_CLASSPATH" ]; then + echo "You must set the ES_CLASSPATH var" >&2 + exit 1 +fi + +# Try to read package config files +if [ -f "/etc/sysconfig/elasticsearch" ]; then + CONF_DIR=/etc/elasticsearch + + . "/etc/sysconfig/elasticsearch" +elif [ -f "/etc/default/elasticsearch" ]; then + CONF_DIR=/etc/elasticsearch + + . "/etc/default/elasticsearch" +fi + +export HOSTNAME=`hostname -s` + +# include x-pack jars in classpath +ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/plugins/x-pack/*" + +# don't let JAVA_TOOL_OPTIONS slip in (e.g. crazy agents in ubuntu) +# works around https://bugs.launchpad.net/ubuntu/+source/jayatana/+bug/1441487 +if [ "x$JAVA_TOOL_OPTIONS" != "x" ]; then + echo "Warning: Ignoring JAVA_TOOL_OPTIONS=$JAVA_TOOL_OPTIONS" + echo "Please pass JVM parameters via ES_JAVA_OPTS instead" + unset JAVA_TOOL_OPTIONS +fi + +# CONF_FILE setting was removed +if [ ! -z "$CONF_FILE" ]; then + echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed." + exit 1 +fi + +declare -a args=("$@") + +if [ -e "$CONF_DIR" ]; then + args=("${args[@]}" -Edefault.path.conf="$CONF_DIR") +fi + +cd "$ES_HOME" > /dev/null +"$JAVA" $ES_JAVA_OPTS -cp "$ES_CLASSPATH" -Des.path.home="$ES_HOME" org.elasticsearch.xpack.security.authc.esnative.ESNativeRealmMigrateTool "${args[@]}" +status=$? +cd - > /dev/null +exit $status diff --git a/elasticsearch/x-pack/security/bin/x-pack/migrate.bat b/elasticsearch/x-pack/security/bin/x-pack/migrate.bat new file mode 100644 index 00000000000..c9f8db9a0dc --- /dev/null +++ b/elasticsearch/x-pack/security/bin/x-pack/migrate.bat @@ -0,0 +1,9 @@ +@echo off + +rem Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +rem or more contributor license agreements. Licensed under the Elastic License; +rem you may not use this file except in compliance with the Elastic License. + +PUSHD "%~dp0" +CALL "%~dp0.in.bat" org.elasticsearch.xpack.security.authc.esnative.ESNativeRealmMigrateTool %* +POPD diff --git a/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/action/user/PutUserRequestBuilder.java b/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/action/user/PutUserRequestBuilder.java index 7b753e9ba97..38888f19c00 100644 --- a/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/action/user/PutUserRequestBuilder.java +++ b/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/action/user/PutUserRequestBuilder.java @@ -79,6 +79,11 @@ public class PutUserRequestBuilder extends ActionRequestBuilder username; + private final OptionSpec password; + private final OptionSpec url; + private final OptionSpec usersToMigrateCsv; + private final OptionSpec rolesToMigrateCsv; + private final OptionSpec esConfigDir; + + public MigrateUserOrRoles() { + super("Migrates users or roles from file to native realm"); + this.username = parser.acceptsAll(Arrays.asList("u", "username"), + "User used to authenticate with Elasticsearch") + .withRequiredArg(); + this.password = parser.acceptsAll(Arrays.asList("p", "password"), + "Password used to authenticate with Elasticsearch") + .withRequiredArg(); + this.url = parser.acceptsAll(Arrays.asList("U", "url"), + "URL of Elasticsearch host") + .withRequiredArg(); + this.usersToMigrateCsv = parser.acceptsAll(Arrays.asList("n", "users"), + "Users to migrate from file to native realm") + .withRequiredArg(); + this.rolesToMigrateCsv = parser.acceptsAll(Arrays.asList("r", "roles"), + "Roles to migrate from file to native realm") + .withRequiredArg(); + this.esConfigDir = parser.acceptsAll(Arrays.asList("c", "config"), + "Configuration directory to use instead of default") + .withRequiredArg(); + } + + // Visible for testing + public OptionParser getParser() { + return this.parser; + } + + @Override + protected void printAdditionalHelp(Terminal terminal) { + terminal.println("This tool migrates file based users[1] and roles[2] to the native realm in"); + terminal.println("elasticsearch, saving the administrator from needing to manually transition"); + terminal.println("them from the file."); + } + + // Visible for testing + @Override + public void execute(Terminal terminal, OptionSet options, Map settings) throws Exception { + terminal.println("starting migration of users and roles..."); + Settings.Builder sb = Settings.builder(); + sb.put(settings); + if (this.esConfigDir != null) { + sb.put("path.conf", this.esConfigDir.value(options)); + } + Settings shieldSettings = sb.build(); + Environment shieldEnv = new Environment(shieldSettings); + importUsers(terminal, shieldSettings, shieldEnv, options); + importRoles(terminal, shieldSettings, shieldEnv, options); + terminal.println("users and roles imported."); + } + + private String postURL(Settings settings, Environment env, String method, String urlString, + OptionSet options, @Nullable String bodyString) throws Exception { + URI uri = new URI(urlString); + URL url = uri.toURL(); + HttpURLConnection conn; + if ("https".equalsIgnoreCase(uri.getScheme())) { + SSLConfiguration.Global globalConfig = new SSLConfiguration.Global(settings); + final ClientSSLService sslService = new ClientSSLService(settings, globalConfig); + sslService.setEnvironment(env); + final HttpsURLConnection httpsConn = (HttpsURLConnection) url.openConnection(); + AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Void run() { + // Requires permission java.lang.RuntimePermission "setFactory"; + httpsConn.setSSLSocketFactory(sslService.sslSocketFactory(settings)); + return null; + } + }); + conn = httpsConn; + } else { + conn = (HttpURLConnection) url.openConnection(); + } + // If using SSL, need a custom service because it's likely a self-signed certificate + conn.setRequestMethod(method); + conn.setReadTimeout(30 * 1000); // 30 second timeout + // Add basic-auth header + conn.setRequestProperty("Authorization", + UsernamePasswordToken.basicAuthHeaderValue(username.value(options), + new SecuredString(password.value(options).toCharArray()))); + conn.setDoOutput(true); // we'll be sending a body + conn.connect(); + if (bodyString != null) { + try (OutputStream out = conn.getOutputStream()) { + out.write(bodyString.getBytes(Charsets.UTF_8)); + } catch (Exception e) { + try { + conn.disconnect(); + } catch (Exception e2) { + // Ignore exceptions if we weren't able to close the connection after an error + } + throw e; + } + } + try (BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream(), Charsets.UTF_8))) { + StringBuilder sb = new StringBuilder(); + String line = null; + while ((line = reader.readLine()) != null) { + sb.append(line); + } + return sb.toString(); + } catch (IOException e) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getErrorStream(), Charsets.UTF_8))) { + StringBuilder sb = new StringBuilder(); + String line = null; + while ((line = reader.readLine()) != null) { + sb.append(line); + } + throw new IOException(sb.toString(), e); + } + } finally { + conn.disconnect(); + } + } + + public Set getUsersThatExist(Terminal terminal, Settings settings, Environment env, OptionSet options) throws Exception { + Set existingUsers = new HashSet<>(); + String allUsersJson = postURL(settings, env, "GET", this.url.value(options) + "/_xpack/security/user/", options, null); + try (XContentParser parser = JsonXContent.jsonXContent.createParser(allUsersJson)) { + XContentParser.Token token = parser.nextToken(); + String userName; + if (token == XContentParser.Token.START_OBJECT) { + while ((token = parser.nextToken()) == XContentParser.Token.FIELD_NAME) { + userName = parser.currentName(); + existingUsers.add(userName); + parser.nextToken(); + parser.skipChildren(); + } + } else { + throw new Exception("failed to retrieve users, expecting an object but got: " + token); + } + } + terminal.println("found existing users: " + existingUsers); + return existingUsers; + } + + public static String createUserJson(String[] roles, char[] password) throws IOException { + XContentBuilder builder = jsonBuilder(); + builder.startObject(); + { + builder.field("password_hash", new String(password)); + builder.startArray("roles"); + for (String role : roles) { + builder.value(role); + } + builder.endArray(); + } + builder.endObject(); + return builder.string(); + } + + public void importUsers(Terminal terminal, Settings settings, Environment env, OptionSet options) { + String usersCsv = usersToMigrateCsv.value(options); + String[] usersToMigrate = (usersCsv != null) ? usersCsv.split(",") : Strings.EMPTY_ARRAY; + Settings esusersSettings = Realms.fileRealmSettings(settings); + Path usersFile = FileUserPasswdStore.resolveFile(esusersSettings, env); + Path usersRolesFile = FileUserRolesStore.resolveFile(esusersSettings, env); + terminal.println("importing users from [" + usersFile + "]..."); + Map userToHashedPW = FileUserPasswdStore.parseFile(usersFile, null); + Map userToRoles = FileUserRolesStore.parseFile(usersRolesFile, null); + Set existingUsers; + try { + existingUsers = getUsersThatExist(terminal, settings, env, options); + } catch (Exception e) { + terminal.println("failed to get users that already exist, skipping user import"); + terminal.println(ExceptionsHelper.stackTrace(e)); + return; + } + if (usersToMigrate.length == 0) { + usersToMigrate = userToHashedPW.keySet().toArray(new String[userToHashedPW.size()]); + } + for (String user : usersToMigrate) { + if (userToHashedPW.containsKey(user) == false) { + terminal.println("no user [" + user + "] found, skipping"); + continue; + } else if (existingUsers.contains(user)) { + terminal.println("user [" + user + "] already exists, skipping"); + continue; + } + terminal.println("migrating user [" + user + "]"); + String reqBody = "n/a"; + try { + reqBody = createUserJson(userToRoles.get(user), userToHashedPW.get(user)); + String resp = postURL(settings, env, "POST", + this.url.value(options) + "/_xpack/security/user/" + user, options, reqBody); + terminal.println(resp); + } catch (Exception e) { + terminal.println("failed to migrate user [" + user + "] with body: " + reqBody); + terminal.println(ExceptionsHelper.stackTrace(e)); + } + } + } + + public Set getRolesThatExist(Terminal terminal, Settings settings, Environment env, OptionSet options) throws Exception { + Set existingRoles = new HashSet<>(); + String allRolesJson = postURL(settings, env, "GET", this.url.value(options) + "/_xpack/security/role/", options, null); + try (XContentParser parser = JsonXContent.jsonXContent.createParser(allRolesJson)) { + XContentParser.Token token = parser.nextToken(); + String roleName; + if (token == XContentParser.Token.START_OBJECT) { + while ((token = parser.nextToken()) == XContentParser.Token.FIELD_NAME) { + roleName = parser.currentName(); + existingRoles.add(roleName); + parser.nextToken(); + parser.skipChildren(); + } + } else { + throw new Exception("failed to retrieve roles, expecting an object but got: " + token); + } + } + terminal.println("found existing roles: " + existingRoles); + return existingRoles; + } + + public static String createRoleJson(RoleDescriptor rd) throws IOException { + XContentBuilder builder = jsonBuilder(); + builder.startObject(); + { + String[] clusterStrings = rd.getClusterPrivileges(); + String[] runAs = rd.getRunAs(); + RoleDescriptor.IndicesPrivileges[] indicesPrivileges = rd.getIndicesPrivileges(); + + if (clusterStrings != null && clusterStrings.length > 0) { + builder.array("cluster", clusterStrings); + } + + if (runAs != null && runAs.length > 0) { + builder.array("run_as", runAs); + } + + if (indicesPrivileges != null && indicesPrivileges.length > 0) { + builder.startArray("indices"); + for (RoleDescriptor.IndicesPrivileges ip : indicesPrivileges) { + builder.startObject(); + { + String[] indices = ip.getIndices(); + String[] privs = ip.getPrivileges(); + String[] fields = ip.getFields(); + BytesReference query = ip.getQuery(); + + if (indices != null && indices.length > 0) { + builder.array("names", indices); + } + + if (privs != null && privs.length > 0) { + builder.array("privileges", privs); + } + + if (fields != null && fields.length > 0) { + builder.array("fields", fields); + } + + if (query != null) { + builder.field("query", query.toUtf8()); + } + } + builder.endObject(); + } + builder.endArray(); + } + } + builder.endObject(); + return builder.string(); + } + + public void importRoles(Terminal terminal, Settings settings, Environment env, OptionSet options) { + String rolesCsv = rolesToMigrateCsv.value(options); + String[] rolesToMigrate = (rolesCsv != null) ? rolesCsv.split(",") : Strings.EMPTY_ARRAY; + Settings esusersSettings = Realms.fileRealmSettings(settings); + Path rolesFile = FileRolesStore.resolveFile(esusersSettings, env).toAbsolutePath(); + terminal.println("importing roles from [" + rolesFile + "]..."); + Map roles = FileRolesStore.parseRoleDescriptors(rolesFile, null, true, Settings.EMPTY); + Set existingRoles; + try { + existingRoles = getRolesThatExist(terminal, settings, env, options); + } catch (Exception e) { + terminal.println("failed to get roles that already exist, skipping role import"); + terminal.println(ExceptionsHelper.stackTrace(e)); + return; + } + if (rolesToMigrate.length == 0) { + rolesToMigrate = roles.keySet().toArray(new String[roles.size()]); + } + for (String roleName : rolesToMigrate) { + if (roles.containsKey(roleName) == false) { + terminal.println("no role [" + roleName + "] found, skipping"); + continue; + } else if (existingRoles.contains(roleName)) { + terminal.println("role [" + roleName + "] already exists, skipping"); + continue; + } + terminal.println("migrating role [" + roleName + "]"); + String reqBody = "n/a"; + try { + reqBody = createRoleJson(roles.get(roleName));; + String resp = postURL(settings, env, "POST", + this.url.value(options) + "/_xpack/security/role/" + roleName, options, reqBody); + terminal.println(resp); + } catch (Exception e) { + terminal.println("failed to migrate role [" + roleName + "] with body: " + reqBody); + terminal.println(ExceptionsHelper.stackTrace(e)); + } + } + } + } +} diff --git a/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java b/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java index cec1f8ad830..1816df5e15e 100644 --- a/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java +++ b/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.security.authz.store; import com.fasterxml.jackson.dataformat.yaml.snakeyaml.error.YAMLException; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.logging.ESLogger; @@ -150,7 +151,55 @@ public class FileRolesStore extends AbstractLifecycleComponent imple return unmodifiableMap(roles); } + public static Map parseRoleDescriptors(Path path, ESLogger logger, + boolean resolvePermission, Settings settings) { + if (logger == null) { + logger = NoOpLogger.INSTANCE; + } + + Map roles = new HashMap<>(); + logger.trace("attempted to read roles file located at [{}]", path.toAbsolutePath()); + if (Files.exists(path)) { + try { + List roleSegments = roleSegments(path); + for (String segment : roleSegments) { + RoleDescriptor rd = parseRoleDescriptor(segment, path, logger, resolvePermission, settings); + if (rd != null) { + roles.put(rd.getName(), rd); + } + } + } catch (IOException ioe) { + logger.error("failed to read roles file [{}]. skipping all roles...", ioe, path.toAbsolutePath()); + } + } + return unmodifiableMap(roles); + } + + @Nullable private static Role parseRole(String segment, Path path, ESLogger logger, boolean resolvePermissions, Settings settings) { + RoleDescriptor descriptor = parseRoleDescriptor(segment, path, logger, resolvePermissions, settings); + + if (descriptor != null) { + String roleName = descriptor.getName(); + // first check if FLS/DLS is enabled on the role... + for (RoleDescriptor.IndicesPrivileges privilege : descriptor.getIndicesPrivileges()) { + if ((privilege.getQuery() != null || privilege.getFields() != null) + && Security.flsDlsEnabled(settings) == false) { + logger.error("invalid role definition [{}] in roles file [{}]. document and field level security is not " + + "enabled. set [{}] to [true] in the configuration file. skipping role...", roleName, path + .toAbsolutePath(), XPackPlugin.featureEnabledSetting(Security.DLS_FLS_FEATURE)); + return null; + } + } + return Role.builder(descriptor).build(); + } else { + return null; + } + } + + @Nullable + private static RoleDescriptor parseRoleDescriptor(String segment, Path path, ESLogger logger, + boolean resolvePermissions, Settings settings) { String roleName = null; try { XContentParser parser = YamlXContent.yamlXContent.createParser(segment); @@ -167,25 +216,13 @@ public class FileRolesStore extends AbstractLifecycleComponent imple } if (resolvePermissions == false) { - return Role.builder(roleName).build(); + return new RoleDescriptor(roleName, null, null, null); } token = parser.nextToken(); if (token == XContentParser.Token.START_OBJECT) { RoleDescriptor descriptor = RoleDescriptor.parse(roleName, parser); - - // first check if FLS/DLS is enabled on the role... - for (RoleDescriptor.IndicesPrivileges privilege : descriptor.getIndicesPrivileges()) { - if ((privilege.getQuery() != null || privilege.getFields() != null) - && Security.flsDlsEnabled(settings) == false) { - logger.error("invalid role definition [{}] in roles file [{}]. document and field level security is not " + - "enabled. set [{}] to [true] in the configuration file. skipping role...", roleName, path - .toAbsolutePath(), XPackPlugin.featureEnabledSetting(Security.DLS_FLS_FEATURE)); - return null; - } - } - - return Role.builder(descriptor).build(); + return descriptor; } else { logger.error("invalid role definition [{}] in roles file [{}]. skipping role...", roleName, path.toAbsolutePath()); return null; diff --git a/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/user/User.java b/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/user/User.java index 57bdd982d15..c22ddc8be27 100644 --- a/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/user/User.java +++ b/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/user/User.java @@ -294,6 +294,7 @@ public class User implements ToXContent { public interface Fields { ParseField USERNAME = new ParseField("username"); ParseField PASSWORD = new ParseField("password"); + ParseField PASSWORD_HASH = new ParseField("password_hash"); ParseField ROLES = new ParseField("roles"); ParseField FULL_NAME = new ParseField("full_name"); ParseField EMAIL = new ParseField("email"); diff --git a/elasticsearch/x-pack/security/src/main/resources/org/elasticsearch/xpack/security/crypto/migrate.help b/elasticsearch/x-pack/security/src/main/resources/org/elasticsearch/xpack/security/crypto/migrate.help new file mode 100644 index 00000000000..5f3a606ea54 --- /dev/null +++ b/elasticsearch/x-pack/security/src/main/resources/org/elasticsearch/xpack/security/crypto/migrate.help @@ -0,0 +1,34 @@ +NAME + + migrate - Migrates elasticsearch file based users and roles to native realm + +SYNOPSIS + + migrate native -U [OPTIONS] + +DESCRIPTION + + This tool migrates file based users[1] and roles[2] to the native realm in + elasticsearch, saving the administrator from needing to manually transition + them from the file. + +OPTIONS + + -U --url URL to connect to to user/role import + + -u --username [Optional] Username for authenticating with Elasticsearch + + -p --password [Optional] Password for authenticating with Elasticsearch + + -n --users [Optional] Users to migrate from file to native realm, + if not specified all users will be migrated + + -r --roles [Optional] Roles to migrate from file to native realm + if not specified all roles will be migrated + + -c --config [Optional] Path to Elasticsearch config directory + +NOTES + + [1] https://www.elastic.co/guide/en/shield/current/setting-up-authentication.html + [2] https://www.elastic.co/guide/en/shield/current/configuring-rbac.html diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java index b7f0583bc06..1e4b9711a34 100644 --- a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java @@ -11,7 +11,10 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xpack.security.InternalClient; import org.elasticsearch.xpack.security.Security; @@ -28,10 +31,12 @@ import org.junit.BeforeClass; import org.junit.Rule; import org.junit.rules.ExternalResource; +import java.net.InetSocketAddress; import java.nio.file.Path; import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; @@ -360,4 +365,19 @@ public abstract class SecurityIntegTestCase extends ESIntegTestCase { public static SecurityClient securityClient(Client client) { return randomBoolean() ? new XPackClient(client).security() : new SecurityClient(client); } -} \ No newline at end of file + + protected String getHttpURL() { + return getHttpURL(false); + } + + protected String getHttpURL(boolean useSSL) { + final NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().get(); + final List nodes = nodeInfos.getNodes(); + assertTrue("there is at least one node", nodes.size() > 0); + NodeInfo ni = randomFrom(nodes); + TransportAddress publishAddress = ni.getHttp().address().publishAddress(); + assertEquals(1, publishAddress.uniqueAddressTypeId()); + InetSocketAddress address = ((InetSocketTransportAddress) publishAddress).address(); + return (useSSL ? "https://" : "http://") + NetworkAddress.format(address.getAddress()) + ":" + address.getPort(); + } +} diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java new file mode 100644 index 00000000000..8a1d609be6c --- /dev/null +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.securit.authc.esnative; + +import com.google.common.base.Charsets; +import joptsimple.OptionParser; +import joptsimple.OptionSet; +import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.ElasticsearchSecurityException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cli.MockTerminal; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.NativeRealmIntegTestCase; +import org.elasticsearch.test.SecuritySettingsSource; +import org.elasticsearch.xpack.security.SecurityTemplateService; +import org.elasticsearch.xpack.security.authc.esnative.ESNativeRealmMigrateTool; +import org.elasticsearch.xpack.security.client.SecurityClient; +import org.elasticsearch.xpack.security.transport.netty.SecurityNettyHttpServerTransport; +import org.junit.BeforeClass; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +/** + * Integration tests for the {@code ESNativeMigrateTool} + */ +public class ESNativeMigrateToolTests extends NativeRealmIntegTestCase { + + // Randomly use SSL (or not) + private static boolean useSSL; + + @BeforeClass + private static void setSSL() { + useSSL = randomBoolean(); + } + + @Override + public Settings nodeSettings(int nodeOrdinal) { + logger.info("--> use SSL? {}", useSSL); + Settings s = Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(NetworkModule.HTTP_ENABLED.getKey(), true) + .put(SecurityNettyHttpServerTransport.SSL_SETTING.getKey(), useSSL) + .build(); + return s; + } + + @Override + protected boolean sslTransportEnabled() { + return useSSL; + } + + private String homePath() throws Exception { + Environment e = internalCluster().getInstances(Environment.class).iterator().next(); + return e.configFile().toAbsolutePath().toString(); + } + + public void testRetrieveUsers() throws Exception { + String home = homePath(); + SecurityClient c = new SecurityClient(client()); + logger.error("--> creating users"); + int numToAdd = randomIntBetween(1,10); + Set addedUsers = new HashSet(numToAdd); + for (int i = 0; i < numToAdd; i++) { + String uname = randomAsciiOfLength(5); + c.preparePutUser(uname, "s3kirt".toCharArray(), "role1", "user").get(); + addedUsers.add(uname); + } + logger.error("--> waiting for .security index"); + ensureGreen(SecurityTemplateService.SECURITY_INDEX_NAME); + + MockTerminal t = new MockTerminal(); + String username = nodeClientUsername(); + String password = new String(nodeClientPassword().utf8Bytes(), Charsets.UTF_8); + String url = getHttpURL(useSSL); + ESNativeRealmMigrateTool.MigrateUserOrRoles muor = new ESNativeRealmMigrateTool.MigrateUserOrRoles(); + Settings sslSettings = + SecuritySettingsSource.getSSLSettingsForStore("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks", + "testnode"); + Settings settings = Settings.builder().put(sslSettings).put("path.home", home).build(); + logger.error("--> retrieving users using URL: {}, home: {}", url, home); + + OptionParser parser = muor.getParser(); + OptionSet options = parser.parse("-u", username, "-p", password, "-U", url, "-c", home); + logger.info("--> options: {}", options.asMap()); + Set users = muor.getUsersThatExist(t, settings, new Environment(settings), options); + logger.info("--> output: \n{}", t.getOutput());; + for (String u : addedUsers) { + assertThat("expected list to contain: " + u, users.contains(u), is(true)); + } + } + + public void testRetrieveRoles() throws Exception { + String home = homePath(); + SecurityClient c = new SecurityClient(client()); + logger.error("--> creating roles"); + int numToAdd = randomIntBetween(1,10); + Set addedRoles = new HashSet(numToAdd); + for (int i = 0; i < numToAdd; i++) { + String rname = randomAsciiOfLength(5); + c.preparePutRole(rname) + .cluster("all", "none") + .runAs("root", "nobody") + .addIndices(new String[]{"index"}, new String[]{"read"}, + new String[]{"body", "title"}, new BytesArray("{\"query\": {\"match_all\": {}}}")) + .get(); + addedRoles.add(rname); + } + logger.error("--> waiting for .security index"); + ensureGreen(SecurityTemplateService.SECURITY_INDEX_NAME); + + MockTerminal t = new MockTerminal(); + String username = nodeClientUsername(); + String password = new String(nodeClientPassword().utf8Bytes(), Charsets.UTF_8); + String url = getHttpURL(useSSL); + ESNativeRealmMigrateTool.MigrateUserOrRoles muor = new ESNativeRealmMigrateTool.MigrateUserOrRoles(); + Settings sslSettings = + SecuritySettingsSource.getSSLSettingsForStore("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.jks", + "testclient"); + Settings settings = Settings.builder().put(sslSettings).put("path.home", home).build(); + logger.error("--> retrieving roles using URL: {}, home: {}", url, home); + + OptionParser parser = muor.getParser(); + OptionSet options = parser.parse("-u", username, "-p", password, "-U", url, "-c", home); + Set roles = muor.getRolesThatExist(t, settings, new Environment(settings), options); + logger.info("--> output: \n{}", t.getOutput());; + for (String r : addedRoles) { + assertThat("expected list to contain: " + r, roles.contains(r), is(true)); + } + } +} diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateToolTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateToolTests.java new file mode 100644 index 00000000000..34ac6f73094 --- /dev/null +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateToolTests.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.shield.authc.esusers.tool; + +import org.elasticsearch.cli.Command; +import org.elasticsearch.cli.CommandTestCase; +import org.elasticsearch.common.Strings; +import org.elasticsearch.xpack.security.authc.esnative.ESNativeRealmMigrateTool; +import org.elasticsearch.xpack.security.authz.RoleDescriptor; +import org.junit.Test; + +import static org.hamcrest.Matchers.equalTo; + +/** + * Unit tests for the {@code ESNativeRealmMigrateTool} + */ +public class ESNativeRealmMigrateToolTests extends CommandTestCase { + + @Override + protected Command newCommand() { + return new ESNativeRealmMigrateTool(); + } + + public void testUserJson() throws Exception { + assertThat(ESNativeRealmMigrateTool.MigrateUserOrRoles.createUserJson(Strings.EMPTY_ARRAY, "hash".toCharArray()), + equalTo("{\"password_hash\":\"hash\",\"roles\":[]}")); + assertThat(ESNativeRealmMigrateTool.MigrateUserOrRoles.createUserJson(new String[]{"role1", "role2"}, "hash".toCharArray()), + equalTo("{\"password_hash\":\"hash\",\"roles\":[\"role1\",\"role2\"]}")); + } + + public void testRoleJson() throws Exception { + RoleDescriptor.IndicesPrivileges ip = RoleDescriptor.IndicesPrivileges.builder() + .indices(new String[]{"i1", "i2", "i3"}) + .privileges(new String[]{"all"}) + .fields(new String[]{"body"}) + .build(); + RoleDescriptor.IndicesPrivileges[] ips = new RoleDescriptor.IndicesPrivileges[1]; + ips[0] = ip; + String[] cluster = Strings.EMPTY_ARRAY; + String[] runAs = Strings.EMPTY_ARRAY; + RoleDescriptor rd = new RoleDescriptor("rolename", cluster, ips, runAs); + assertThat(ESNativeRealmMigrateTool.MigrateUserOrRoles.createRoleJson(rd), + equalTo("{\"indices\":[{\"names\":[\"i1\",\"i2\",\"i3\"],\"privileges\":[\"all\"],\"fields\":[\"body\"]}]}")); + + } + +} From 2d0055de3548830ca7e74ef0846d56d842617747 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 28 Jun 2016 14:06:53 -0700 Subject: [PATCH 02/39] Make script service a cluster state listener Original commit: elastic/x-pack-elasticsearch@dc763f3d66ac23432ad4a897a17dd72ef7ff7f04 --- .../elasticsearch/messy/tests/MessyTestUtils.java | 5 +---- .../elasticsearch/messy/tests/SearchInputIT.java | 4 ++-- .../messy/tests/SearchTransformIT.java | 4 ++-- .../smoketest/WatcherTemplateTests.java | 4 +--- .../xpack/common/ScriptServiceProxy.java | 10 ++++------ .../xpack/watcher/test/WatcherTestUtils.java | 13 +------------ 6 files changed, 11 insertions(+), 29 deletions(-) diff --git a/elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/MessyTestUtils.java b/elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/MessyTestUtils.java index dee4d8ae66b..f4584212067 100644 --- a/elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/MessyTestUtils.java +++ b/elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/MessyTestUtils.java @@ -39,11 +39,8 @@ public final class MessyTestUtils { ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singleton(groovyScriptEngineService)); ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Arrays.asList(ScriptServiceProxy.INSTANCE)); - ClusterService clusterService = Mockito.mock(ClusterService.class); - Mockito.when(clusterService.state()).thenReturn(ClusterState.builder(new ClusterName("_name")).build()); ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); return ScriptServiceProxy.of(new ScriptService(settings, new Environment(settings), - new ResourceWatcherService(settings, tp), scriptEngineRegistry, scriptContextRegistry, scriptSettings), - clusterService); + new ResourceWatcherService(settings, tp), scriptEngineRegistry, scriptContextRegistry, scriptSettings)); } } diff --git a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java index bb5a50d38d3..2e7600ca438 100644 --- a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java +++ b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java @@ -362,7 +362,7 @@ public class SearchInputIT extends ESIntegTestCase { protected WatcherSearchTemplateService watcherSearchTemplateService() { String master = internalCluster().getMasterName(); return new WatcherSearchTemplateService(internalCluster().clusterService(master).getSettings(), - ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class, master), internalCluster().clusterService(master)), + ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class, master)), internalCluster().getInstance(IndicesQueriesRegistry.class, master), internalCluster().getInstance(AggregatorParsers.class, master), internalCluster().getInstance(Suggesters.class, master) @@ -370,7 +370,7 @@ public class SearchInputIT extends ESIntegTestCase { } protected ScriptServiceProxy scriptService() { - return ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class), internalCluster().clusterService()); + return ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class)); } private XContentSource toXContentSource(SearchInput.Result result) throws IOException { diff --git a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java index 7cb66e537fc..2b469f17527 100644 --- a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java +++ b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java @@ -519,7 +519,7 @@ public class SearchTransformIT extends ESIntegTestCase { protected WatcherSearchTemplateService watcherSearchTemplateService() { String master = internalCluster().getMasterName(); return new WatcherSearchTemplateService(internalCluster().clusterService(master).getSettings(), - ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class, master), internalCluster().clusterService(master)), + ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class, master)), internalCluster().getInstance(IndicesQueriesRegistry.class, master), internalCluster().getInstance(AggregatorParsers.class, master), internalCluster().getInstance(Suggesters.class, master) @@ -527,7 +527,7 @@ public class SearchTransformIT extends ESIntegTestCase { } protected ScriptServiceProxy scriptService() { - return ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class), internalCluster().clusterService()); + return ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class)); } private static Map doc(String date, String value) { diff --git a/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/java/org/elasticsearch/smoketest/WatcherTemplateTests.java b/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/java/org/elasticsearch/smoketest/WatcherTemplateTests.java index 5475529e2d8..3a90293ff75 100644 --- a/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/java/org/elasticsearch/smoketest/WatcherTemplateTests.java +++ b/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/java/org/elasticsearch/smoketest/WatcherTemplateTests.java @@ -58,9 +58,7 @@ public class WatcherTemplateTests extends ESTestCase { ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, registry); ScriptService scriptService = new ScriptService(setting, environment, resourceWatcherService, scriptEngineRegistry, registry, scriptSettings); - ClusterService clusterService = Mockito.mock(ClusterService.class); - Mockito.when(clusterService.state()).thenReturn(ClusterState.builder(new ClusterName("_name")).build()); - engine = new DefaultTextTemplateEngine(Settings.EMPTY, ScriptServiceProxy.of(scriptService, clusterService)); + engine = new DefaultTextTemplateEngine(Settings.EMPTY, ScriptServiceProxy.of(scriptService)); } public void testEscaping() throws Exception { diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/common/ScriptServiceProxy.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/common/ScriptServiceProxy.java index acd4b8939d7..8f224cd5508 100644 --- a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/common/ScriptServiceProxy.java +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/common/ScriptServiceProxy.java @@ -26,13 +26,11 @@ public class ScriptServiceProxy { private final ScriptService service; private final SecurityContext securityContext; - private final ClusterService clusterService; @Inject - public ScriptServiceProxy(ScriptService service, SecurityContext securityContext, ClusterService clusterService) { + public ScriptServiceProxy(ScriptService service, SecurityContext securityContext) { this.service = service; this.securityContext = securityContext; - this.clusterService = clusterService; } public CompiledScript compile(Script script) { @@ -41,7 +39,7 @@ public class ScriptServiceProxy { public CompiledScript compile(org.elasticsearch.script.Script script, Map compileParams) { return securityContext.executeAs(XPackUser.INSTANCE, () -> - service.compile(script, WatcherScriptContext.CTX, compileParams, clusterService.state())); + service.compile(script, WatcherScriptContext.CTX, compileParams)); } public ExecutableScript executable(CompiledScript compiledScript, Map vars) { @@ -64,7 +62,7 @@ public class ScriptServiceProxy { /** * Factory helper method for testing. */ - public static ScriptServiceProxy of(ScriptService service, ClusterService clusterService) { - return new ScriptServiceProxy(service, SecurityContext.Insecure.INSTANCE, clusterService); + public static ScriptServiceProxy of(ScriptService service) { + return new ScriptServiceProxy(service, SecurityContext.Insecure.INSTANCE); } } diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java index 3cff684748e..dd85faa8f06 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java @@ -256,11 +256,8 @@ public final class WatcherTestUtils { ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.emptyList()); ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); - ClusterService clusterService = Mockito.mock(ClusterService.class); - Mockito.when(clusterService.state()).thenReturn(ClusterState.builder(new ClusterName("_name")).build()); return ScriptServiceProxy.of(new ScriptService(settings, new Environment(settings), - new ResourceWatcherService(settings, tp), scriptEngineRegistry, scriptContextRegistry, scriptSettings), - clusterService); + new ResourceWatcherService(settings, tp), scriptEngineRegistry, scriptContextRegistry, scriptSettings)); } public static SearchType getRandomSupportedSearchType() { @@ -270,12 +267,4 @@ public final class WatcherTestUtils { SearchType.DFS_QUERY_THEN_FETCH, SearchType.DFS_QUERY_AND_FETCH); } - - public static boolean areJsonEquivalent(String json1, String json2) throws IOException { - XContentParser parser1 = XContentHelper.createParser(new BytesArray(json1.getBytes(StandardCharsets.UTF_8))); - XContentParser parser2 = XContentHelper.createParser(new BytesArray(json2.getBytes(StandardCharsets.UTF_8))); - Map map1 = parser1.map(); - Map map2 = parser2.map(); - return map1.equals(map2); - } } From a06f4a02fd2a3666f44419f8c48b36ba1043f53a Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 28 Jun 2016 10:57:24 +0200 Subject: [PATCH 03/39] Rename Marvel packages and directories to Monitoring related to elastic/elasticsearch#2383 Original commit: elastic/x-pack-elasticsearch@7f4001a031cbd9c48715e45f1ed9197b6537c67e --- .../qa/smoke-test-plugins-ssl/build.gradle | 2 +- elasticsearch/x-pack/build.gradle | 2 +- .../xpack/graph/test/GraphTests.java | 3 +- .../AbstractLicensesIntegrationTestCase.java | 2 +- .../plugin/LicensesTransportTests.java | 2 +- .../xpack/TribeTransportTestCase.java | 2 +- .../xpack/monitoring}/MonitoredSystem.java | 2 +- .../xpack/monitoring}/Monitoring.java | 19 +++--- .../monitoring}/MonitoringFeatureSet.java | 6 +- .../xpack/monitoring}/MonitoringLicensee.java | 4 +- .../xpack/monitoring}/MonitoringModule.java | 6 +- .../xpack/monitoring}/MonitoringSettings.java | 5 +- .../action/MonitoringBulkAction.java | 2 +- .../monitoring}/action/MonitoringBulkDoc.java | 4 +- .../action/MonitoringBulkRequest.java | 4 +- .../action/MonitoringBulkRequestBuilder.java | 2 +- .../action/MonitoringBulkResponse.java | 2 +- .../monitoring}/action/MonitoringIndex.java | 2 +- .../action/TransportMonitoringBulkAction.java | 8 +-- .../xpack/monitoring}/agent/AgentService.java | 16 ++--- .../agent/collector/AbstractCollector.java | 12 ++-- .../agent/collector/Collector.java | 4 +- .../agent/collector/CollectorModule.java | 16 ++--- .../cluster/ClusterInfoMonitoringDoc.java | 4 +- .../cluster/ClusterStateCollector.java | 16 ++--- .../cluster/ClusterStateMonitoringDoc.java | 4 +- .../ClusterStateNodeMonitoringDoc.java | 4 +- .../cluster/ClusterStatsCollector.java | 13 ++-- .../cluster/ClusterStatsMonitoringDoc.java | 4 +- .../cluster/DiscoveryNodeMonitoringDoc.java | 4 +- .../indices/IndexRecoveryCollector.java | 12 ++-- .../indices/IndexRecoveryMonitoringDoc.java | 4 +- .../indices/IndexStatsCollector.java | 12 ++-- .../indices/IndexStatsMonitoringDoc.java | 4 +- .../indices/IndicesStatsCollector.java | 12 ++-- .../indices/IndicesStatsMonitoringDoc.java | 4 +- .../collector/node/NodeStatsCollector.java | 12 ++-- .../node/NodeStatsMonitoringDoc.java | 4 +- .../collector/shards/ShardMonitoringDoc.java | 4 +- .../collector/shards/ShardsCollector.java | 12 ++-- .../agent/exporter/ExportBulk.java | 2 +- .../agent/exporter/ExportException.java | 2 +- .../monitoring}/agent/exporter/Exporter.java | 4 +- .../agent/exporter/ExporterModule.java | 8 +-- .../monitoring}/agent/exporter/Exporters.java | 6 +- .../agent/exporter/MonitoringDoc.java | 2 +- .../exporter/MonitoringTemplateUtils.java} | 6 +- .../agent/exporter/http/HttpExporter.java | 16 ++--- .../exporter/http/HttpExporterUtils.java | 2 +- .../agent/exporter/local/LocalBulk.java | 12 ++-- .../agent/exporter/local/LocalExporter.java | 14 ++--- .../resolver/MonitoringIndexNameResolver.java | 20 +++--- .../agent/resolver/ResolversRegistry.java | 58 +++++++++-------- .../bulk/MonitoringBulkDataResolver.java | 6 +- .../MonitoringBulkTimestampedResolver.java | 8 +-- .../resolver/cluster/ClusterInfoResolver.java | 6 +- .../cluster/ClusterStateNodeResolver.java | 8 +-- .../cluster/ClusterStateResolver.java | 8 +-- .../cluster/ClusterStatsResolver.java | 8 +-- .../cluster/DiscoveryNodeResolver.java | 6 +- .../indices/IndexRecoveryResolver.java | 8 +-- .../resolver/indices/IndexStatsResolver.java | 8 +-- .../indices/IndicesStatsResolver.java | 8 +-- .../resolver/node/NodeStatsResolver.java | 8 +-- .../agent/resolver/shards/ShardsResolver.java | 8 +-- .../monitoring}/cleaner/CleanerService.java | 6 +- .../monitoring}/client/MonitoringClient.java | 10 +-- .../client/MonitoringClientModule.java | 2 +- .../rest/MonitoringRestHandler.java | 2 +- .../rest/action/RestMonitoringBulkAction.java | 8 +-- .../monitoring}/support/VersionUtils.java | 2 +- .../src/main/resources/monitoring-data.json | 0 .../src/main/resources/monitoring-es.json | 0 .../src/main/resources/monitoring-kibana.json | 0 .../stats/IndicesStatsResponseTestUtils.java | 0 .../xpack/monitoring}/MonitoringF.java | 2 +- .../MonitoringFeatureSetTests.java | 10 +-- .../MonitoringPluginClientTests.java} | 7 +-- .../monitoring/MonitoringPluginTests.java} | 20 +++--- .../monitoring/MonitoringSettingsTests.java} | 5 +- .../monitoring}/MonitoringTribeTests.java | 8 +-- .../action/MonitoringBulkDocTests.java | 4 +- .../action/MonitoringBulkRequestTests.java | 2 +- .../action/MonitoringBulkResponseTests.java | 4 +- .../action/MonitoringBulkTests.java | 12 ++-- .../action/MonitoringIndexTests.java | 2 +- .../TransportMonitoringBulkActionTests.java | 13 ++-- .../collector/AbstractCollectorTestCase.java | 9 +-- .../cluster/ClusterStateCollectorTests.java | 58 ++++++++--------- .../cluster/ClusterStatsCollectorTests.java | 53 ++++++++-------- .../indices/IndexRecoveryCollectorTests.java | 39 ++++++------ .../indices/IndexStatsCollectorTests.java | 39 ++++++------ .../indices/IndicesStatsCollectorTests.java | 42 ++++++------- .../node/NodeStatsCollectorTests.java | 40 ++++++------ .../shards/ShardsCollectorTests.java | 34 +++++----- .../AbstractExporterTemplateTestCase.java | 22 +++---- .../agent/exporter/ExportersTests.java | 10 +-- .../agent/exporter/MonitoringDocTests.java | 2 +- .../MonitoringTemplateUtilsTests.java} | 10 +-- .../http/HttpExporterTemplateTests.java | 6 +- .../exporter/http/HttpExporterTests.java | 44 ++++++------- .../exporter/http/HttpExporterUtilsTests.java | 2 +- .../local/LocalExporterTemplateTests.java | 6 +- .../exporter/local/LocalExporterTests.java | 63 +++++++++---------- .../agent/resolver/DataResolverTests.java | 10 +-- .../MonitoringIndexNameResolverTestCase.java | 48 +++++++------- .../resolver/TimestampedResolverTests.java | 18 +++--- .../bulk/MonitoringBulkDataResolverTests.java | 16 ++--- ...onitoringBulkTimestampedResolverTests.java | 14 ++--- .../cluster/ClusterInfoResolverTests.java | 16 ++--- .../resolver/cluster/ClusterInfoTests.java | 26 ++++---- .../ClusterStateNodeResolverTests.java | 14 ++--- .../cluster/ClusterStateResolverTests.java | 14 ++--- .../resolver/cluster/ClusterStateTests.java | 42 ++++++------- .../cluster/ClusterStatsResolverTests.java | 22 +++---- .../resolver/cluster/ClusterStatsTests.java | 18 +++--- .../indices/IndexRecoveryResolverTests.java | 14 ++--- .../resolver/indices/IndexRecoveryTests.java | 22 +++---- .../indices/IndexStatsResolverTests.java | 14 ++--- .../resolver/indices/IndexStatsTests.java | 20 +++--- .../indices/IndicesStatsResolverTests.java | 14 ++--- .../resolver/indices/IndicesStatsTests.java | 20 +++--- .../node/DiscoveryNodeResolverTests.java | 16 ++--- .../resolver/node/MultiNodesStatsTests.java | 16 ++--- .../resolver/node/NodeStatsResolverTests.java | 14 ++--- .../agent/resolver/node/NodeStatsTests.java | 22 +++---- .../resolver/shards/ShardsResolverTests.java | 16 ++--- .../agent/resolver/shards/ShardsTests.java | 26 ++++---- .../settings/MonitoringSettingsTests.java} | 22 +++---- .../AbstractIndicesCleanerTestCase.java | 24 +++---- .../cleaner/CleanerServiceTests.java | 6 +- .../local/LocalIndicesCleanerTests.java | 6 +- .../license/LicenseIntegrationTests.java | 8 +-- .../license/MonitoringLicenseeTests.java} | 6 +- .../MonitoringInternalClientTests.java} | 8 +-- .../MonitoringSettingsFilterTests.java} | 8 +-- .../support/VersionUtilsTests.java | 2 +- .../test/MonitoringIntegTestCase.java} | 60 +++++++++--------- .../src/test/resources/log4j.properties | 2 +- .../src/test/resources/monitoring-test.json | 0 .../api/xpack.monitoring.bulk.json | 2 +- .../test/monitoring/bulk/10_basic.yaml | 0 .../x-pack/security/config/x-pack/roles.yml | 6 +- .../security/authz/permission/KibanaRole.java | 2 +- .../test/SecuritySettingsSource.java | 2 +- .../authz/permission/KibanaRoleTests.java | 2 +- .../authz/permission/KibanaUserRoleTests.java | 2 +- .../org/elasticsearch/transport/actions | 1 + .../security/authz/store/default_roles.yml | 6 +- .../org/elasticsearch/xpack/XPackClient.java | 2 +- .../org/elasticsearch/xpack/XPackPlugin.java | 4 +- .../watcher/WatcherPluginDisableTests.java | 2 +- .../AbstractWatcherIntegrationTestCase.java | 2 +- .../WatcherSettingsFilterTests.java | 6 +- 154 files changed, 873 insertions(+), 886 deletions(-) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/MonitoredSystem.java (95%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/Monitoring.java (83%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/MonitoringFeatureSet.java (95%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/MonitoringLicensee.java (97%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/MonitoringModule.java (87%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/MonitoringSettings.java (98%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/action/MonitoringBulkAction.java (95%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/action/MonitoringBulkDoc.java (94%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/action/MonitoringBulkRequest.java (96%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/action/MonitoringBulkRequestBuilder.java (95%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/action/MonitoringBulkResponse.java (98%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/action/MonitoringIndex.java (97%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/action/TransportMonitoringBulkAction.java (96%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/AgentService.java (93%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/AbstractCollector.java (91%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/Collector.java (77%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/CollectorModule.java (68%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/cluster/ClusterInfoMonitoringDoc.java (90%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/cluster/ClusterStateCollector.java (87%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/cluster/ClusterStateMonitoringDoc.java (87%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/cluster/ClusterStateNodeMonitoringDoc.java (85%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/cluster/ClusterStatsCollector.java (92%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/cluster/ClusterStatsMonitoringDoc.java (84%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/cluster/DiscoveryNodeMonitoringDoc.java (82%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/indices/IndexRecoveryCollector.java (90%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/indices/IndexRecoveryMonitoringDoc.java (85%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/indices/IndexStatsCollector.java (91%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/indices/IndexStatsMonitoringDoc.java (83%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/indices/IndicesStatsCollector.java (90%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/indices/IndicesStatsMonitoringDoc.java (84%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/node/NodeStatsCollector.java (93%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/node/NodeStatsMonitoringDoc.java (93%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/shards/ShardMonitoringDoc.java (87%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/collector/shards/ShardsCollector.java (91%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/exporter/ExportBulk.java (98%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/exporter/ExportException.java (97%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/exporter/Exporter.java (96%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/exporter/ExporterModule.java (87%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/exporter/Exporters.java (97%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/exporter/MonitoringDoc.java (99%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtils.java => monitoring/src/main/java/org/elasticsearch/xpack/monitoring/agent/exporter/MonitoringTemplateUtils.java} (86%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/exporter/http/HttpExporter.java (98%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/exporter/http/HttpExporterUtils.java (95%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/exporter/local/LocalBulk.java (90%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/exporter/local/LocalExporter.java (96%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/resolver/MonitoringIndexNameResolver.java (91%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/resolver/ResolversRegistry.java (67%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/resolver/bulk/MonitoringBulkDataResolver.java (83%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/resolver/bulk/MonitoringBulkTimestampedResolver.java (80%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/resolver/cluster/ClusterInfoResolver.java (92%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/resolver/cluster/ClusterStateNodeResolver.java (81%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/resolver/cluster/ClusterStateResolver.java (86%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/resolver/cluster/ClusterStatsResolver.java (88%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/resolver/cluster/DiscoveryNodeResolver.java (88%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/resolver/indices/IndexRecoveryResolver.java (88%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/resolver/indices/IndexStatsResolver.java (94%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/resolver/indices/IndicesStatsResolver.java (89%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/resolver/node/NodeStatsResolver.java (95%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/agent/resolver/shards/ShardsResolver.java (90%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/cleaner/CleanerService.java (98%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/client/MonitoringClient.java (83%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/client/MonitoringClientModule.java (90%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/rest/MonitoringRestHandler.java (93%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/rest/action/RestMonitoringBulkAction.java (92%) rename elasticsearch/x-pack/{marvel/src/main/java/org/elasticsearch/marvel => monitoring/src/main/java/org/elasticsearch/xpack/monitoring}/support/VersionUtils.java (96%) rename elasticsearch/x-pack/{marvel => monitoring}/src/main/resources/monitoring-data.json (100%) rename elasticsearch/x-pack/{marvel => monitoring}/src/main/resources/monitoring-es.json (100%) rename elasticsearch/x-pack/{marvel => monitoring}/src/main/resources/monitoring-kibana.json (100%) rename elasticsearch/x-pack/{marvel => monitoring}/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTestUtils.java (100%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/MonitoringF.java (97%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/MonitoringFeatureSetTests.java (94%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel/MarvelPluginClientTests.java => monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringPluginClientTests.java} (88%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel/MarvelPluginTests.java => monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringPluginTests.java} (86%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel/MarvelSettingsTests.java => monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringSettingsTests.java} (94%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/MonitoringTribeTests.java (86%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/action/MonitoringBulkDocTests.java (96%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/action/MonitoringBulkRequestTests.java (99%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/action/MonitoringBulkResponseTests.java (96%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/action/MonitoringBulkTests.java (93%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/action/MonitoringIndexTests.java (97%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/action/TransportMonitoringBulkActionTests.java (96%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/collector/AbstractCollectorTestCase.java (96%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/collector/cluster/ClusterStateCollectorTests.java (78%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/collector/cluster/ClusterStatsCollectorTests.java (73%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/collector/indices/IndexRecoveryCollectorTests.java (84%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/collector/indices/IndexStatsCollectorTests.java (86%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/collector/indices/IndicesStatsCollectorTests.java (84%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/collector/node/NodeStatsCollectorTests.java (73%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/collector/shards/ShardsCollectorTests.java (85%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/exporter/AbstractExporterTemplateTestCase.java (91%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/exporter/ExportersTests.java (98%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/exporter/MonitoringDocTests.java (99%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtilsTests.java => monitoring/src/test/java/org/elasticsearch/xpack/monitoring/agent/exporter/MonitoringTemplateUtilsTests.java} (70%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/exporter/http/HttpExporterTemplateTests.java (96%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/exporter/http/HttpExporterTests.java (93%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/exporter/http/HttpExporterUtilsTests.java (98%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/exporter/local/LocalExporterTemplateTests.java (87%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/exporter/local/LocalExporterTests.java (81%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/DataResolverTests.java (84%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/MonitoringIndexNameResolverTestCase.java (77%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/TimestampedResolverTests.java (83%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/bulk/MonitoringBulkDataResolverTests.java (82%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/bulk/MonitoringBulkTimestampedResolverTests.java (84%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/cluster/ClusterInfoResolverTests.java (85%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/cluster/ClusterInfoTests.java (87%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/cluster/ClusterStateNodeResolverTests.java (80%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/cluster/ClusterStateResolverTests.java (84%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/cluster/ClusterStateTests.java (84%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/cluster/ClusterStatsResolverTests.java (89%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/cluster/ClusterStatsTests.java (83%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/indices/IndexRecoveryResolverTests.java (83%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/indices/IndexRecoveryTests.java (87%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/indices/IndexStatsResolverTests.java (89%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/indices/IndexStatsTests.java (85%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/indices/IndicesStatsResolverTests.java (90%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/indices/IndicesStatsTests.java (84%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/node/DiscoveryNodeResolverTests.java (78%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/node/MultiNodesStatsTests.java (91%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/node/NodeStatsResolverTests.java (92%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/node/NodeStatsTests.java (84%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/shards/ShardsResolverTests.java (90%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/agent/resolver/shards/ShardsTests.java (85%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel/agent/settings/MarvelSettingsTests.java => monitoring/src/test/java/org/elasticsearch/xpack/monitoring/agent/settings/MonitoringSettingsTests.java} (93%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/cleaner/AbstractIndicesCleanerTestCase.java (90%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/cleaner/CleanerServiceTests.java (97%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/cleaner/local/LocalIndicesCleanerTests.java (90%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/license/LicenseIntegrationTests.java (95%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel/license/MarvelLicenseeTests.java => monitoring/src/test/java/org/elasticsearch/xpack/monitoring/license/MonitoringLicenseeTests.java} (96%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel/security/MarvelInternalClientTests.java => monitoring/src/test/java/org/elasticsearch/xpack/monitoring/security/MonitoringInternalClientTests.java} (93%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel/security/MarvelSettingsFilterTests.java => monitoring/src/test/java/org/elasticsearch/xpack/monitoring/security/MonitoringSettingsFilterTests.java} (93%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel => monitoring/src/test/java/org/elasticsearch/xpack/monitoring}/support/VersionUtilsTests.java (97%) rename elasticsearch/x-pack/{marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java => monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java} (93%) rename elasticsearch/x-pack/{marvel => monitoring}/src/test/resources/log4j.properties (85%) rename elasticsearch/x-pack/{marvel => monitoring}/src/test/resources/monitoring-test.json (100%) rename elasticsearch/x-pack/{marvel => monitoring}/src/test/resources/rest-api-spec/api/xpack.monitoring.bulk.json (89%) rename elasticsearch/x-pack/{marvel => monitoring}/src/test/resources/rest-api-spec/test/monitoring/bulk/10_basic.yaml (100%) diff --git a/elasticsearch/qa/smoke-test-plugins-ssl/build.gradle b/elasticsearch/qa/smoke-test-plugins-ssl/build.gradle index 05fc100d3f9..e2c6ace1b8b 100644 --- a/elasticsearch/qa/smoke-test-plugins-ssl/build.gradle +++ b/elasticsearch/qa/smoke-test-plugins-ssl/build.gradle @@ -174,7 +174,7 @@ integTest { setupCommand 'setupTestUser', 'bin/x-pack/users', 'useradd', 'test_user', '-p', 'changeme', '-r', 'superuser' - setupCommand 'setupMarvelUser', + setupCommand 'setupMonitoringUser', 'bin/x-pack/users', 'useradd', 'monitoring_agent', '-p', 'changeme', '-r', 'remote_monitoring_agent' // Required to detect that the monitoring agent service has started diff --git a/elasticsearch/x-pack/build.gradle b/elasticsearch/x-pack/build.gradle index 391172ce60b..05e30057874 100644 --- a/elasticsearch/x-pack/build.gradle +++ b/elasticsearch/x-pack/build.gradle @@ -57,7 +57,7 @@ dependencies { // we keep the source directories in the original structure of split plugins, // in order to facilitate backports to 2.x. TODO: remove after 5.0 release -for (String module : ['', 'license-plugin/', 'security/', 'watcher/', 'marvel/', 'graph/']) { +for (String module : ['', 'license-plugin/', 'security/', 'watcher/', 'monitoring/', 'graph/']) { sourceSets { main { java.srcDir("${module}src/main/java") diff --git a/elasticsearch/x-pack/graph/src/test/java/org/elasticsearch/xpack/graph/test/GraphTests.java b/elasticsearch/x-pack/graph/src/test/java/org/elasticsearch/xpack/graph/test/GraphTests.java index 26c7e77394d..a4ad8cdb434 100644 --- a/elasticsearch/x-pack/graph/src/test/java/org/elasticsearch/xpack/graph/test/GraphTests.java +++ b/elasticsearch/x-pack/graph/src/test/java/org/elasticsearch/xpack/graph/test/GraphTests.java @@ -13,14 +13,13 @@ import org.elasticsearch.common.settings.Settings.Builder; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.ScriptQueryBuilder; -import org.elasticsearch.marvel.Monitoring; +import org.elasticsearch.xpack.monitoring.Monitoring; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.script.AbstractSearchScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.NativeScriptFactory; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.xpack.security.Security; import org.elasticsearch.test.ESSingleNodeTestCase; diff --git a/elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/AbstractLicensesIntegrationTestCase.java b/elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/AbstractLicensesIntegrationTestCase.java index e3758326bd5..76b6f8055dd 100644 --- a/elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/AbstractLicensesIntegrationTestCase.java +++ b/elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/AbstractLicensesIntegrationTestCase.java @@ -23,7 +23,7 @@ import org.elasticsearch.license.plugin.core.LicenseState; import org.elasticsearch.license.plugin.core.LicensesManagerService; import org.elasticsearch.license.plugin.core.LicensesMetaData; import org.elasticsearch.license.plugin.core.LicensesStatus; -import org.elasticsearch.marvel.Monitoring; +import org.elasticsearch.xpack.monitoring.Monitoring; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xpack.security.Security; import org.elasticsearch.test.ESIntegTestCase; diff --git a/elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/LicensesTransportTests.java b/elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/LicensesTransportTests.java index 77d76020f88..997e14062a7 100644 --- a/elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/LicensesTransportTests.java +++ b/elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/LicensesTransportTests.java @@ -19,7 +19,7 @@ import org.elasticsearch.license.plugin.action.put.PutLicenseAction; import org.elasticsearch.license.plugin.action.put.PutLicenseRequestBuilder; import org.elasticsearch.license.plugin.action.put.PutLicenseResponse; import org.elasticsearch.license.plugin.core.LicensesStatus; -import org.elasticsearch.marvel.Monitoring; +import org.elasticsearch.xpack.monitoring.Monitoring; import org.elasticsearch.node.Node; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xpack.security.Security; diff --git a/elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/xpack/TribeTransportTestCase.java b/elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/xpack/TribeTransportTestCase.java index 9cb295c296c..962ca7f5e37 100644 --- a/elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/xpack/TribeTransportTestCase.java +++ b/elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/xpack/TribeTransportTestCase.java @@ -20,7 +20,7 @@ import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing; -import org.elasticsearch.marvel.Monitoring; +import org.elasticsearch.xpack.monitoring.Monitoring; import org.elasticsearch.node.Node; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xpack.security.Security; diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MonitoredSystem.java b/elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoredSystem.java similarity index 95% rename from elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MonitoredSystem.java rename to elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoredSystem.java index 45ddf37802f..227c63d84b7 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MonitoredSystem.java +++ b/elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoredSystem.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.marvel; +package org.elasticsearch.xpack.monitoring; import java.util.Locale; diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/Monitoring.java b/elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java similarity index 83% rename from elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/Monitoring.java rename to elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java index cf8ccd6fdbe..3ca63653af9 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/Monitoring.java +++ b/elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.marvel; +package org.elasticsearch.xpack.monitoring; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; @@ -11,14 +11,15 @@ import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.marvel.action.MonitoringBulkAction; -import org.elasticsearch.marvel.action.TransportMonitoringBulkAction; -import org.elasticsearch.marvel.agent.AgentService; -import org.elasticsearch.marvel.agent.collector.CollectorModule; -import org.elasticsearch.marvel.agent.exporter.ExporterModule; -import org.elasticsearch.marvel.cleaner.CleanerService; -import org.elasticsearch.marvel.client.MonitoringClientModule; -import org.elasticsearch.marvel.rest.action.RestMonitoringBulkAction; +import org.elasticsearch.xpack.XPackPlugin; +import org.elasticsearch.xpack.monitoring.action.MonitoringBulkAction; +import org.elasticsearch.xpack.monitoring.action.TransportMonitoringBulkAction; +import org.elasticsearch.xpack.monitoring.agent.AgentService; +import org.elasticsearch.xpack.monitoring.agent.collector.CollectorModule; +import org.elasticsearch.xpack.monitoring.agent.exporter.ExporterModule; +import org.elasticsearch.xpack.monitoring.cleaner.CleanerService; +import org.elasticsearch.xpack.monitoring.client.MonitoringClientModule; +import org.elasticsearch.xpack.monitoring.rest.action.RestMonitoringBulkAction; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.xpack.XPackPlugin; diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MonitoringFeatureSet.java b/elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringFeatureSet.java similarity index 95% rename from elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MonitoringFeatureSet.java rename to elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringFeatureSet.java index c28118df0dd..717c40dbb02 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MonitoringFeatureSet.java +++ b/elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringFeatureSet.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.marvel; +package org.elasticsearch.xpack.monitoring; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.inject.Inject; @@ -12,9 +12,9 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.marvel.agent.exporter.Exporter; -import org.elasticsearch.marvel.agent.exporter.Exporters; import org.elasticsearch.xpack.XPackFeatureSet; +import org.elasticsearch.xpack.monitoring.agent.exporter.Exporter; +import org.elasticsearch.xpack.monitoring.agent.exporter.Exporters; import java.io.IOException; import java.util.HashMap; diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MonitoringLicensee.java b/elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringLicensee.java similarity index 97% rename from elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MonitoringLicensee.java rename to elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringLicensee.java index f63ff611b16..058bff4c18c 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MonitoringLicensee.java +++ b/elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringLicensee.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.marvel; +package org.elasticsearch.xpack.monitoring; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; @@ -17,7 +17,7 @@ import org.elasticsearch.license.plugin.core.Licensee; import org.elasticsearch.license.plugin.core.LicenseeRegistry; /** - * {@code MarvelLicensee} determines whether certain features of Monitoring are enabled or disabled. + * {@code MonitoringLicensee} determines whether certain features of Monitoring are enabled or disabled. *

* Once the license expires, the agent will stop: *