Merge branch 'master' into license-checking/reporting-ux

Original commit: elastic/x-pack-elasticsearch@be886ba2a4

elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/DateUtils.java

@@ -13,11 +13,11 @@ import org.joda.time.format.ISODateTimeFormat;
 
 public class DateUtils {
 
-    private final static FormatDateTimeFormatter formatDateOnlyFormatter = Joda.forPattern("yyyy-MM-dd");
+    private static final FormatDateTimeFormatter formatDateOnlyFormatter = Joda.forPattern("yyyy-MM-dd");
 
-    private final static DateTimeFormatter dateOnlyFormatter = formatDateOnlyFormatter.parser().withZoneUTC();
+    private static final DateTimeFormatter dateOnlyFormatter = formatDateOnlyFormatter.parser().withZoneUTC();
 
-    private final static DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime().withZoneUTC();
+    private static final DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime().withZoneUTC();
 
     public static long endOfTheDay(String date) {
         try {

elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/License.java

@@ -29,9 +29,9 @@ import java.util.Locale;
  * Provides serialization/deserialization & validation methods for license object
  */
 public class License implements ToXContent {
-    public final static int VERSION_START = 1;
+    public static final int VERSION_START = 1;
-    public final static int VERSION_NO_FEATURE_TYPE = 2;
+    public static final int VERSION_NO_FEATURE_TYPE = 2;
-    public final static int VERSION_CURRENT = VERSION_NO_FEATURE_TYPE;
+    public static final int VERSION_CURRENT = VERSION_NO_FEATURE_TYPE;
 
     /**
      * XContent param name to deserialize license(s) with
@@ -51,7 +51,7 @@ public class License implements ToXContent {
      */
     public static final String LICENSE_VERSION_MODE = "license_version";
 
-    public final static Comparator<License> LATEST_ISSUE_DATE_FIRST = new Comparator<License>() {
+    public static final Comparator<License> LATEST_ISSUE_DATE_FIRST = new Comparator<License>() {
         @Override
         public int compare(License right, License left) {
             return Long.compare(left.issueDate(), right.issueDate());
@@ -506,7 +506,7 @@ public class License implements ToXContent {
         return result;
     }
 
-    public final static class Fields {
+    public static final class Fields {
         public static final String STATUS = "status";
         public static final String UID = "uid";
         public static final String TYPE = "type";

elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/LicenseVerifier.java

@@ -5,6 +5,8 @@
  */
 package org.elasticsearch.license.core;
 
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefIterator;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -51,7 +53,11 @@ public class LicenseVerifier {
             license.toXContent(contentBuilder, new ToXContent.MapParams(Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true")));
             Signature rsa = Signature.getInstance("SHA512withRSA");
             rsa.initVerify(CryptUtils.readEncryptedPublicKey(encryptedPublicKeyData));
-            rsa.update(contentBuilder.bytes().toBytes());
+            BytesRefIterator iterator = contentBuilder.bytes().iterator();
+            BytesRef ref;
+            while((ref = iterator.next()) != null) {
+                rsa.update(ref.bytes, ref.offset, ref.length);
+            }
             return rsa.verify(signedContent)
                     && Arrays.equals(Base64.getEncoder().encode(encryptedPublicKeyData), signatureHash);
         } catch (IOException | NoSuchAlgorithmException | SignatureException | InvalidKeyException e) {

elasticsearch/license/base/src/test/java/org/elasticsearch/license/core/TestUtils.java

@@ -25,9 +25,9 @@ import static org.hamcrest.core.IsEqual.equalTo;
 
 public class TestUtils {
 
-    private final static FormatDateTimeFormatter formatDateTimeFormatter = Joda.forPattern("yyyy-MM-dd");
+    private static final FormatDateTimeFormatter formatDateTimeFormatter = Joda.forPattern("yyyy-MM-dd");
-    private final static DateMathParser dateMathParser = new DateMathParser(formatDateTimeFormatter);
+    private static final DateMathParser dateMathParser = new DateMathParser(formatDateTimeFormatter);
-    private final static DateTimeFormatter dateTimeFormatter = formatDateTimeFormatter.printer();
+    private static final DateTimeFormatter dateTimeFormatter = formatDateTimeFormatter.printer();
 
     public static String dateMathString(String time, final long now) {
         return dateTimeFormatter.print(dateMathParser.parse(time, new Callable<Long>() {

elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/LicenseSigner.java

@@ -5,6 +5,9 @@
  */
 package org.elasticsearch.license.licensor;
 
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefIterator;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -30,7 +33,7 @@ import java.util.Collections;
  */
 public class LicenseSigner {
 
-    private final static int MAGIC_LENGTH = 13;
+    private static final int MAGIC_LENGTH = 13;
 
     private final Path publicKeyPath;
 
@@ -55,7 +58,11 @@ public class LicenseSigner {
         try {
             final Signature rsa = Signature.getInstance("SHA512withRSA");
             rsa.initSign(CryptUtils.readEncryptedPrivateKey(Files.readAllBytes(privateKeyPath)));
-            rsa.update(contentBuilder.bytes().toBytes());
+            final BytesRefIterator iterator = contentBuilder.bytes().iterator();
+            BytesRef ref;
+            while((ref = iterator.next()) != null) {
+                rsa.update(ref.bytes, ref.offset, ref.length);
+            }
             signedContent = rsa.sign();
         } catch (InvalidKeyException | IOException | NoSuchAlgorithmException | SignatureException e) {
             throw new IllegalStateException(e);

elasticsearch/license/licensor/src/test/java/org/elasticsearch/license/licensor/TestUtils.java

@@ -36,9 +36,9 @@ public class TestUtils {
     public static final String PUBLIC_KEY_RESOURCE = "/public.key";
     public static final String PRIVATE_KEY_RESOURCE = "/private.key";
 
-    private final static FormatDateTimeFormatter formatDateTimeFormatter = Joda.forPattern("yyyy-MM-dd");
+    private static final FormatDateTimeFormatter formatDateTimeFormatter = Joda.forPattern("yyyy-MM-dd");
-    private final static DateMathParser dateMathParser = new DateMathParser(formatDateTimeFormatter);
+    private static final DateMathParser dateMathParser = new DateMathParser(formatDateTimeFormatter);
-    private final static DateTimeFormatter dateTimeFormatter = formatDateTimeFormatter.printer();
+    private static final DateTimeFormatter dateTimeFormatter = formatDateTimeFormatter.printer();
 
     public static String dumpLicense(License license) throws Exception {
         XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);

elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/MessyTestUtils.java

@@ -39,11 +39,8 @@ public final class MessyTestUtils {
         ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singleton(groovyScriptEngineService));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Arrays.asList(ScriptServiceProxy.INSTANCE));
 
-        ClusterService clusterService = Mockito.mock(ClusterService.class);
-        Mockito.when(clusterService.state()).thenReturn(ClusterState.builder(new ClusterName("_name")).build());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         return  ScriptServiceProxy.of(new ScriptService(settings, new Environment(settings),
-                new ResourceWatcherService(settings, tp), scriptEngineRegistry, scriptContextRegistry, scriptSettings),
+                new ResourceWatcherService(settings, tp), scriptEngineRegistry, scriptContextRegistry, scriptSettings));
-                clusterService);
     }
 }

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java

@@ -93,7 +93,7 @@ public class SearchInputIT extends ESIntegTestCase {
         return types;
     }
 
-    private final static String TEMPLATE_QUERY = "{\"query\":{\"bool\":{\"must\":{\"match\":{\"event_type\":{\"query\":\"a\"," +
+    private static final String TEMPLATE_QUERY = "{\"query\":{\"bool\":{\"must\":{\"match\":{\"event_type\":{\"query\":\"a\"," +
             "\"type\":\"boolean\"}}},\"filter\":{\"range\":{\"_timestamp\":" +
             "{\"from\":\"{{ctx.trigger.scheduled_time}}||-{{seconds_param}}\",\"to\":\"{{ctx.trigger.scheduled_time}}\"," +
             "\"include_lower\":true,\"include_upper\":true}}}}}}";
@@ -362,7 +362,7 @@ public class SearchInputIT extends ESIntegTestCase {
     protected WatcherSearchTemplateService watcherSearchTemplateService() {
         String master = internalCluster().getMasterName();
         return new WatcherSearchTemplateService(internalCluster().clusterService(master).getSettings(),
-                ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class, master), internalCluster().clusterService(master)),
+                ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class, master)),
                 internalCluster().getInstance(IndicesQueriesRegistry.class, master),
                 internalCluster().getInstance(AggregatorParsers.class, master),
                 internalCluster().getInstance(Suggesters.class, master)
@@ -370,7 +370,7 @@ public class SearchInputIT extends ESIntegTestCase {
     }
 
     protected ScriptServiceProxy scriptService() {
-        return ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class), internalCluster().clusterService());
+        return ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class));
     }
 
     private XContentSource toXContentSource(SearchInput.Result result) throws IOException {

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java

@@ -519,7 +519,7 @@ public class SearchTransformIT extends ESIntegTestCase {
     protected WatcherSearchTemplateService watcherSearchTemplateService() {
         String master = internalCluster().getMasterName();
         return new WatcherSearchTemplateService(internalCluster().clusterService(master).getSettings(),
-                ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class, master), internalCluster().clusterService(master)),
+                ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class, master)),
                 internalCluster().getInstance(IndicesQueriesRegistry.class, master),
                 internalCluster().getInstance(AggregatorParsers.class, master),
                 internalCluster().getInstance(Suggesters.class, master)
@@ -527,7 +527,7 @@ public class SearchTransformIT extends ESIntegTestCase {
     }
 
     protected ScriptServiceProxy scriptService() {
-        return ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class), internalCluster().clusterService());
+        return ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class));
     }
 
     private static Map<String, Object> doc(String date, String value) {

elasticsearch/qa/security-migrate-tests/build.gradle

@@ -0,0 +1,32 @@
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+  testCompile project(path: ':x-plugins:elasticsearch:x-pack', configuration: 'runtime')
+}
+
+integTest {
+  cluster {
+    setting 'script.inline', 'true'
+    plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack')
+    extraConfigFile 'x-pack/roles.yml', 'roles.yml'
+    [
+      test_admin: 'superuser',
+      transport_user: 'superuser',
+      existing: 'superuser',
+      bob: 'actual_role'
+    ].each { String user, String role ->
+      setupCommand 'setupUser#' + user,
+                   'bin/x-pack/users', 'useradd', user, '-p', 'changeme', '-r', role
+    }
+    waitCondition = { node, ant ->
+      File tmpFile = new File(node.cwd, 'wait.success')
+      ant.get(src: "http://${node.httpUri()}",
+              dest: tmpFile.toString(),
+              username: 'test_admin',
+              password: 'changeme',
+              ignoreerrors: true,
+              retries: 10)
+      return tmpFile.exists()
+    }
+  }
+}

elasticsearch/qa/security-migrate-tests/roles.yml

@@ -0,0 +1,17 @@
+actual_role:
+  run_as: [ "joe" ]
+  cluster:
+    - monitor
+  indices:
+    - names: [ "index1", "index2" ]
+      privileges: [ "read", "write", "create_index", "indices:admin/refresh" ]
+      fields:
+         - foo
+         - bar
+      query:
+        bool:
+          must_not:
+            match:
+              hidden: true
+    - names: "*"
+      privileges: [ "read" ]

elasticsearch/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java

@@ -0,0 +1,123 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Requests;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.action.role.GetRolesResponse;
+import org.elasticsearch.xpack.security.action.user.GetUsersResponse;
+import org.elasticsearch.xpack.security.action.user.PutUserResponse;
+import org.elasticsearch.xpack.security.authc.esnative.ESNativeRealmMigrateTool;
+import org.elasticsearch.xpack.security.authc.support.SecuredString;
+import org.elasticsearch.xpack.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.security.client.SecurityClient;
+import org.elasticsearch.xpack.security.user.User;
+import org.junit.Before;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.hamcrest.Matchers.containsString;
+
+/**
+ * Integration tests for the {@code migrate} shell command
+ */
+public class MigrateToolIT extends MigrateToolTestCase {
+
+    @Before
+    public void setupUpTest() throws Exception {
+        Client client = getClient();
+        SecurityClient c = new SecurityClient(client);
+
+        // Add an existing user so the tool will skip it
+        PutUserResponse pur = c.preparePutUser("existing", "s3kirt".toCharArray(), "role1", "user").get();
+        assertTrue(pur.created());
+    }
+
+    private static String[] args(String command) {
+        if (!Strings.hasLength(command)) {
+            return Strings.EMPTY_ARRAY;
+        }
+        return command.split("\\s+");
+    }
+
+    public void testRunMigrateTool() throws Exception {
+        Settings settings = Settings.builder()
+                .put("path.home", createTempDir().toAbsolutePath().toString())
+                .build();
+        String integHome = System.getProperty("tests.config.dir");
+        logger.info("--> HOME: {}", integHome);
+        // Cluster should already be up
+        String url = "http://" + getHttpURL();
+        logger.info("--> using URL: {}", url);
+        MockTerminal t = new MockTerminal();
+        ESNativeRealmMigrateTool.MigrateUserOrRoles muor = new ESNativeRealmMigrateTool.MigrateUserOrRoles();
+        OptionParser parser = muor.getParser();
+        OptionSet options = parser.parse("-u", "test_admin", "-p", "changeme", "-U", url, "-c", integHome);
+        muor.execute(t, options, settings.getAsMap());
+
+        logger.info("--> output:\n{}", t.getOutput());
+
+        Client client = getClient();
+        SecurityClient c = new SecurityClient(client);
+
+        // Check that the migrated user can be retrieved
+        GetUsersResponse resp = c.prepareGetUsers("bob").get();
+        assertTrue("user 'bob' should exist", resp.hasUsers());
+        User bob = resp.users()[0];
+        assertEquals(bob.principal(), "bob");
+        assertArrayEquals(bob.roles(), new String[]{"actual_role"});
+
+        // Make sure the existing user did not change
+        resp = c.prepareGetUsers("existing").get();
+        assertTrue("user should exist", resp.hasUsers());
+        User existing = resp.users()[0];
+        assertEquals(existing.principal(), "existing");
+        assertArrayEquals(existing.roles(), new String[]{"role1", "user"});
+
+        // Make sure the "actual_role" made it in and is correct
+        GetRolesResponse roleResp = c.prepareGetRoles().names("actual_role").get();
+        assertTrue("role should exist", roleResp.hasRoles());
+        RoleDescriptor rd = roleResp.roles()[0];
+        assertNotNull(rd);
+        assertEquals(rd.getName(), "actual_role");
+        assertArrayEquals(rd.getClusterPrivileges(), new String[]{"monitor"});
+        assertArrayEquals(rd.getRunAs(), new String[]{"joe"});
+        RoleDescriptor.IndicesPrivileges[] ips = rd.getIndicesPrivileges();
+        assertEquals(ips.length, 2);
+        for (RoleDescriptor.IndicesPrivileges ip : ips) {
+            if (Arrays.equals(ip.getIndices(), new String[]{"index1", "index2"})) {
+                assertArrayEquals(ip.getPrivileges(), new String[]{"read", "write", "create_index", "indices:admin/refresh"});
+                assertArrayEquals(ip.getFields(), new String[]{"foo", "bar"});
+                assertNotNull(ip.getQuery());
+                assertThat(ip.getQuery().utf8ToString(), containsString("{\"bool\":{\"must_not\":{\"match\":{\"hidden\":true}}}}"));
+            } else {
+                assertArrayEquals(ip.getIndices(), new String[]{"*"});
+                assertArrayEquals(ip.getPrivileges(), new String[]{"read"});
+                assertArrayEquals(ip.getFields(), null);
+                assertNull(ip.getQuery());
+            }
+        }
+
+        // Check that bob can access the things the "actual_role" says he can
+        String token = basicAuthHeaderValue("bob", new SecuredString("changeme".toCharArray()));
+        // Create "index1" index and try to search from it as "bob"
+        client.filterWithHeader(Collections.singletonMap("Authorization", token)).admin().indices().prepareCreate("index1").get();
+        SearchResponse searchResp = client.filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("index1").get();
+    }
+}

elasticsearch/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolTestCase.java

@@ -0,0 +1,177 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.node.internal.InternalSettingsPreparer;
+import org.elasticsearch.xpack.security.Security;
+import org.elasticsearch.xpack.XPackPlugin;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.file.Path;
+import java.util.Locale;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiOfLength;
+import static org.hamcrest.Matchers.notNullValue;
+
+/**
+ * {@link MigrateToolTestCase} is an abstract base class to run integration
+ * tests against an external Elasticsearch Cluster.
+ * <p>
+ * You can define a list of transport addresses from where you can reach your cluster
+ * by setting "tests.cluster" system property. It defaults to "localhost:9300".
+ * <p>
+ * All tests can be run from maven using mvn install as maven will start an external cluster first.
+ * <p>
+ * If you want to debug this module from your IDE, then start an external cluster by yourself
+ * then run JUnit. If you changed the default port, set "tests.cluster=localhost:PORT" when running
+ * your test.
+ */
+@LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose")
+public abstract class MigrateToolTestCase extends LuceneTestCase {
+
+    /**
+     * Key used to eventually switch to using an external cluster and provide its transport addresses
+     */
+    public static final String TESTS_CLUSTER = "tests.cluster";
+
+    /**
+     * Key used to eventually switch to using an external cluster and provide its transport addresses
+     */
+    public static final String TESTS_HTTP_CLUSTER = "tests.rest.cluster";
+
+    /**
+     * Defaults to localhost:9300
+     */
+    public static final String TESTS_CLUSTER_DEFAULT = "localhost:9300";
+
+    protected static final ESLogger logger = ESLoggerFactory.getLogger(MigrateToolTestCase.class.getName());
+
+    private static final AtomicInteger counter = new AtomicInteger();
+    private static Client client;
+    private static String clusterAddresses;
+    private static String clusterHttpAddresses;
+
+    private static Client startClient(Path tempDir, TransportAddress... transportAddresses) {
+        logger.info("--> Starting Elasticsearch Java TransportClient {}, {}", transportAddresses, tempDir);
+
+        Settings clientSettings = Settings.builder()
+                .put("cluster.name", "qa_migrate_tests_" + counter.getAndIncrement())
+                .put("client.transport.ignore_cluster_name", true)
+                .put("path.home", tempDir)
+                .put(Security.USER_SETTING.getKey(), "transport_user:changeme")
+                .put("node.mode", "network") // we require network here!
+                .build();
+
+        TransportClient.Builder transportClientBuilder = TransportClient.builder()
+                .addPlugin(XPackPlugin.class)
+                .settings(clientSettings);
+        TransportClient client = transportClientBuilder.build().addTransportAddresses(transportAddresses);
+
+        logger.info("--> Elasticsearch Java TransportClient started");
+
+        Exception clientException = null;
+        try {
+            ClusterHealthResponse health = client.admin().cluster().prepareHealth().get();
+            logger.info("--> connected to [{}] cluster which is running [{}] node(s).",
+                    health.getClusterName(), health.getNumberOfNodes());
+        } catch (Exception e) {
+            clientException = e;
+        }
+
+        assumeNoException("Sounds like your cluster is not running at " + clusterAddresses, clientException);
+
+        return client;
+    }
+
+    private static Client startClient() throws UnknownHostException {
+        String[] stringAddresses = clusterAddresses.split(",");
+        TransportAddress[] transportAddresses = new TransportAddress[stringAddresses.length];
+        int i = 0;
+        for (String stringAddress : stringAddresses) {
+            int lastColon = stringAddress.lastIndexOf(":");
+            if (lastColon == -1) {
+                throw new IllegalArgumentException("address [" + clusterAddresses + "] not valid");
+            }
+            String ip = stringAddress.substring(0, lastColon);
+            String port = stringAddress.substring(lastColon + 1);
+            try {
+                transportAddresses[i++] = new InetSocketTransportAddress(InetAddress.getByName(ip), Integer.valueOf(port));
+            } catch (NumberFormatException e) {
+                throw new IllegalArgumentException("port is not valid, expected number but was [" + port + "]");
+            }
+        }
+        return startClient(createTempDir(), transportAddresses);
+    }
+
+    public static Client getClient() {
+        if (client == null) {
+            try {
+                client = startClient();
+            } catch (UnknownHostException e) {
+                logger.error("could not start the client", e);
+            }
+            assertThat(client, notNullValue());
+        }
+        return client;
+    }
+
+    public static String getHttpURL() {
+        return clusterHttpAddresses;
+    }
+
+    @BeforeClass
+    public static void initializeSettings() throws UnknownHostException {
+        String port = System.getProperty("integ.http.port");
+        clusterAddresses = System.getProperty(TESTS_CLUSTER);
+        clusterHttpAddresses = System.getProperty(TESTS_HTTP_CLUSTER);
+        if (clusterAddresses == null || clusterAddresses.isEmpty()) {
+            throw new UnknownHostException("unable to get a cluster address");
+        }
+    }
+
+    @AfterClass
+    public static void stopTransportClient() {
+        if (client != null) {
+            client.close();
+            client = null;
+        }
+    }
+
+    @Before
+    public void defineIndexName() {
+        doClean();
+    }
+
+    @After
+    public void cleanIndex() {
+        doClean();
+    }
+
+    private void doClean() {
+        if (client != null) {
+            try {
+                client.admin().indices().prepareDelete("_all").get();
+            } catch (Exception e) {
+                // We ignore this cleanup exception
+            }
+        }
+    }
+}

elasticsearch/qa/smoke-test-graph-with-security/src/test/java/org/elasticsearch/smoketest/GraphWithSecurityIT.java

@@ -22,8 +22,8 @@ import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordTok
 
 public class GraphWithSecurityIT extends ESRestTestCase {
 
-    private final static String TEST_ADMIN_USERNAME = "test_admin";
+    private static final String TEST_ADMIN_USERNAME = "test_admin";
-    private final static String TEST_ADMIN_PASSWORD = "changeme";
+    private static final String TEST_ADMIN_PASSWORD = "changeme";
 
     public GraphWithSecurityIT(@Name("yaml") RestTestCandidate testCandidate) {
         super(testCandidate);

elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/java/org/elasticsearch/smoketest/WatcherTemplateTests.java

@@ -58,9 +58,7 @@ public class WatcherTemplateTests extends ESTestCase {
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, registry);
         ScriptService scriptService = new ScriptService(setting, environment, resourceWatcherService, scriptEngineRegistry,
                 registry, scriptSettings);
-        ClusterService clusterService = Mockito.mock(ClusterService.class);
+        engine = new DefaultTextTemplateEngine(Settings.EMPTY, ScriptServiceProxy.of(scriptService));
-        Mockito.when(clusterService.state()).thenReturn(ClusterState.builder(new ClusterName("_name")).build());
-        engine = new DefaultTextTemplateEngine(Settings.EMPTY, ScriptServiceProxy.of(scriptService, clusterService));
     }
 
     public void testEscaping() throws Exception {

elasticsearch/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/WatcherWithSecurityIT.java

@@ -27,8 +27,8 @@ import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordTok
 
 public class WatcherWithSecurityIT extends ESRestTestCase {
 
-    private final static String TEST_ADMIN_USERNAME = "test_admin";
+    private static final String TEST_ADMIN_USERNAME = "test_admin";
-    private final static String TEST_ADMIN_PASSWORD = "changeme";
+    private static final String TEST_ADMIN_PASSWORD = "changeme";
 
     public WatcherWithSecurityIT(@Name("yaml") RestTestCandidate testCandidate) {
         super(testCandidate);

elasticsearch/x-pack/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java

@@ -16,7 +16,7 @@ import java.util.Map;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -66,8 +66,8 @@ public class RestGraphAction extends BaseRestHandler {
     public static final ParseField TERM_FIELD = new ParseField("term");
 
     @Inject
-    public RestGraphAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry indicesQueriesRegistry) {
+    public RestGraphAction(Settings settings, RestController controller, IndicesQueriesRegistry indicesQueriesRegistry) {
-        super(settings, client);
+        super(settings);
         // @deprecated TODO need to add deprecation support as per https://github.com/elastic/x-plugins/issues/1760#issuecomment-217507517
         controller.registerHandler(GET, "/{index}/_graph/explore", this);
         controller.registerHandler(POST, "/{index}/_graph/explore", this);
@@ -82,7 +82,7 @@ public class RestGraphAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws IOException {
         GraphExploreRequest graphRequest = new GraphExploreRequest(Strings.splitStringByCommaToArray(request.param("index")));
         graphRequest.indicesOptions(IndicesOptions.fromRequest(request, graphRequest.indicesOptions()));
         graphRequest.routing(request.param("routing"));

elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/core/AbstractLicenseeComponent.java

@@ -14,7 +14,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
 /**
  * A supporting base class for injectable Licensee components.
  */
-public abstract class AbstractLicenseeComponent<T extends AbstractLicenseeComponent<T>> extends AbstractLifecycleComponent<T>
+public abstract class AbstractLicenseeComponent<T extends AbstractLicenseeComponent<T>> extends AbstractLifecycleComponent
         implements Licensee {
 
     private final String id;

elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/core/LicenseUtils.java

@@ -10,7 +10,7 @@ import org.elasticsearch.rest.RestStatus;
 
 public class LicenseUtils {
 
-    public final static String EXPIRED_FEATURE_HEADER = "es.license.expired.feature";
+    public static final String EXPIRED_FEATURE_HEADER = "es.license.expired.feature";
 
     /**
      * Exception to be thrown when a feature action requires a valid license, but license
@@ -21,7 +21,7 @@ public class LicenseUtils {
      */
     public static ElasticsearchSecurityException newComplianceException(String feature) {
         ElasticsearchSecurityException e = new ElasticsearchSecurityException("current license is non-compliant for [{}]",
-                RestStatus.UNAUTHORIZED, feature);
+                RestStatus.FORBIDDEN, feature);
         e.addHeader(EXPIRED_FEATURE_HEADER, feature);
         return e;
     }

elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/core/LicensesMetaData.java

@@ -9,6 +9,7 @@ import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -186,7 +187,7 @@ public class LicensesMetaData extends AbstractDiffable<MetaData.Custom> implemen
                 XContentBuilder contentBuilder = XContentFactory.contentBuilder(XContentType.JSON);
                 license.toXContent(contentBuilder,
                         new ToXContent.MapParams(Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true")));
-                streamOutput.writeString(Base64.getEncoder().encodeToString(encrypt(contentBuilder.bytes().toBytes())));
+                streamOutput.writeString(Base64.getEncoder().encodeToString(encrypt(BytesReference.toBytes(contentBuilder.bytes()))));
             }
         } else {
             if (license == LICENSE_TOMBSTONE) {
@@ -238,7 +239,7 @@ public class LicensesMetaData extends AbstractDiffable<MetaData.Custom> implemen
         return new LicensesMetaData(license);
     }
 
-    private final static class Fields {
+    private static final class Fields {
         private static final String SIGNED_LICENCES = "signed_licenses";
         private static final String TRIAL_LICENSES = "trial_licenses";
         private static final String LICENSE = "license";

elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/core/LicensesService.java

@@ -79,7 +79,7 @@ import java.util.concurrent.atomic.AtomicReference;
  * Registered listeners are notified using {@link #notifyAndSchedule(LicensesMetaData)}
  */
 @Singleton
-public class LicensesService extends AbstractLifecycleComponent<LicensesService> implements ClusterStateListener, LicensesManagerService,
+public class LicensesService extends AbstractLifecycleComponent implements ClusterStateListener, LicensesManagerService,
         LicenseeRegistry {
 
     public static final String REGISTER_TRIAL_LICENSE_ACTION_NAME = "internal:plugin/license/cluster/register_trial_license";
@@ -643,11 +643,11 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
         }
     }
 
-    public static abstract class ExpirationCallback {
+    public abstract static class ExpirationCallback {
 
         public enum Orientation {PRE, POST}
 
-        public static abstract class Pre extends ExpirationCallback {
+        public abstract static class Pre extends ExpirationCallback {
 
             /**
              * Callback schedule prior to license expiry
@@ -677,7 +677,7 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
             }
         }
 
-        public static abstract class Post extends ExpirationCallback {
+        public abstract static class Post extends ExpirationCallback {
 
             /**
              * Callback schedule after license expiry

elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/core/TrialLicense.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.license.plugin.core;
 
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -32,7 +33,7 @@ public class TrialLicense {
         try {
             XContentBuilder contentBuilder = XContentFactory.contentBuilder(XContentType.JSON);
             spec.toXContent(contentBuilder, new ToXContent.MapParams(Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true")));
-            byte[] encrypt = encrypt(contentBuilder.bytes().toBytes());
+            byte[] encrypt = encrypt(BytesReference.toBytes(contentBuilder.bytes()));
             byte[] bytes = new byte[4 + 4 + encrypt.length];
             ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
             // always generate license version -VERSION_CURRENT

elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/rest/RestDeleteLicenseAction.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.license.plugin.rest;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.plugin.action.delete.DeleteLicenseAction;
@@ -21,13 +21,13 @@ import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 public class RestDeleteLicenseAction extends BaseRestHandler {
 
     @Inject
-    public RestDeleteLicenseAction(Settings settings, RestController controller, Client client) {
+    public RestDeleteLicenseAction(Settings settings, RestController controller) {
-        super(settings, client);
+        super(settings);
         controller.registerHandler(DELETE, "/_xpack/license", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         client.admin().cluster().execute(DeleteLicenseAction.INSTANCE, new DeleteLicenseRequest(), new AcknowledgedRestListener<>(channel));
     }
 }

elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/rest/RestGetLicenseAction.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.license.plugin.rest;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -32,8 +32,8 @@ import static org.elasticsearch.rest.RestStatus.OK;
 public class RestGetLicenseAction extends BaseRestHandler {
 
     @Inject
-    public RestGetLicenseAction(Settings settings, RestController controller, Client client) {
+    public RestGetLicenseAction(Settings settings, RestController controller) {
-        super(settings, client);
+        super(settings);
         controller.registerHandler(GET, "/_xpack/license", this);
     }
 
@@ -44,7 +44,7 @@ public class RestGetLicenseAction extends BaseRestHandler {
      * The licenses are sorted by latest issue_date
      */
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final Map<String, String> overrideParams = new HashMap<>(2);
         overrideParams.put(License.REST_VIEW_MODE, "true");
         overrideParams.put(License.LICENSE_VERSION_MODE, String.valueOf(License.VERSION_CURRENT));

elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/rest/RestPutLicenseAction.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.license.plugin.rest;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -28,16 +28,16 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT;
 public class RestPutLicenseAction extends BaseRestHandler {
 
     @Inject
-    public RestPutLicenseAction(Settings settings, RestController controller, Client client) {
+    public RestPutLicenseAction(Settings settings, RestController controller) {
-        super(settings, client);
+        super(settings);
         controller.registerHandler(PUT, "/_xpack/license", this);
         controller.registerHandler(POST, "/_xpack/license", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         PutLicenseRequest putLicenseRequest = new PutLicenseRequest();
-        putLicenseRequest.license(request.content().toUtf8());
+        putLicenseRequest.license(request.content().utf8ToString());
         putLicenseRequest.acknowledge(request.paramAsBoolean("acknowledge", false));
         client.admin().cluster().execute(PutLicenseAction.INSTANCE, putLicenseRequest,
                 new RestBuilderListener<PutLicenseResponse>(channel) {

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/PutLicenseResponseTests.java

@@ -5,8 +5,8 @@
  */
 package org.elasticsearch.license.plugin;
 
-import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -16,7 +16,6 @@ import org.elasticsearch.license.plugin.core.LicensesStatus;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
@@ -70,7 +69,7 @@ public class PutLicenseResponseTests extends ESTestCase {
         // write it out
         response.writeTo(output);
 
-        ByteBufferStreamInput input = new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytes()));
+        StreamInput input = output.bytes().streamInput();
 
         // read it back in
         response.readFrom(input);

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/TestUtils.java

@@ -48,8 +48,8 @@ import static org.junit.Assert.assertThat;
 
 public class TestUtils {
 
-    private final static FormatDateTimeFormatter formatDateTimeFormatter = Joda.forPattern("yyyy-MM-dd");
+    private static final FormatDateTimeFormatter formatDateTimeFormatter = Joda.forPattern("yyyy-MM-dd");
-    private final static DateMathParser dateMathParser = new DateMathParser(formatDateTimeFormatter);
+    private static final DateMathParser dateMathParser = new DateMathParser(formatDateTimeFormatter);
 
     public static long dateMath(String time, final long now) {
         return dateMathParser.parse(time, new Callable<Long>() {

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/TrialLicenseTests.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.license.plugin;
 
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -96,7 +97,7 @@ public class TrialLicenseTests extends ESTestCase {
         try {
             XContentBuilder contentBuilder = XContentFactory.contentBuilder(XContentType.JSON);
             spec.toXContent(contentBuilder, new ToXContent.MapParams(Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true")));
-            byte[] encrypt = encrypt(contentBuilder.bytes().toBytes());
+            byte[] encrypt = encrypt(BytesReference.toBytes(contentBuilder.bytes()));
             byte[] bytes = new byte[4 + 4 + encrypt.length];
             ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
             byteBuffer.putInt(-spec.version())

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/consumer/EagerLicenseRegistrationConsumerPlugin.java

@@ -16,7 +16,7 @@ import org.elasticsearch.common.settings.Settings;
  */
 public class EagerLicenseRegistrationConsumerPlugin extends TestConsumerPluginBase {
 
-    public final static String NAME = "test_consumer_plugin_1";
+    public static final String NAME = "test_consumer_plugin_1";
 
     @Inject
     public EagerLicenseRegistrationConsumerPlugin(Settings settings) {

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/consumer/TestPluginServiceBase.java

@@ -21,7 +21,7 @@ import org.elasticsearch.license.plugin.core.LicensesService;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
-public abstract class TestPluginServiceBase extends AbstractLifecycleComponent<TestPluginServiceBase>
+public abstract class TestPluginServiceBase extends AbstractLifecycleComponent
         implements ClusterStateListener, Licensee {
 
     private LicensesService licensesClientService;

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesMetaDataSerializationTests.java

@@ -9,8 +9,9 @@ import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
-import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -24,7 +25,6 @@ import org.elasticsearch.license.plugin.Licensing;
 import org.elasticsearch.license.plugin.TestUtils;
 import org.elasticsearch.test.ESTestCase;
 
-import java.nio.ByteBuffer;
 import java.util.Base64;
 import java.util.Collections;
 import java.util.UUID;
@@ -42,9 +42,7 @@ public class LicensesMetaDataSerializationTests extends ESTestCase {
         builder.startObject("licenses");
         licensesMetaData.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
-        byte[] serializedBytes = builder.bytes().toBytes();
+        LicensesMetaData licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(builder.bytes());
-
-        LicensesMetaData licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(serializedBytes);
         assertThat(licensesMetaDataFromXContent.getLicense(), equalTo(license));
     }
 
@@ -90,9 +88,7 @@ public class LicensesMetaDataSerializationTests extends ESTestCase {
         builder.startObject("licenses");
         licensesMetaData.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
-        byte[] serializedBytes = builder.bytes().toBytes();
+        LicensesMetaData licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(builder.bytes());
-
-        LicensesMetaData licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(serializedBytes);
         assertThat(licensesMetaDataFromXContent.getLicense(), equalTo(trialLicense));
     }
 
@@ -113,13 +109,13 @@ public class LicensesMetaDataSerializationTests extends ESTestCase {
         builder.startArray("trial_licenses");
         XContentBuilder contentBuilder = XContentFactory.contentBuilder(XContentType.JSON);
         trialLicense.toXContent(contentBuilder, new ToXContent.MapParams(Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true")));
-        builder.value(Base64.getEncoder().encodeToString(encrypt(contentBuilder.bytes().toBytes())));
+        builder.value(Base64.getEncoder().encodeToString(encrypt(BytesReference.toBytes(contentBuilder.bytes()))));
         builder.endArray();
         builder.startArray("signed_licenses");
         builder.endArray();
         builder.endObject();
         builder.endObject();
-        LicensesMetaData licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(builder.bytes().toBytes());
+        LicensesMetaData licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(builder.bytes());
         assertThat(licensesMetaDataFromXContent.getLicense(), equalTo(trialLicense));
 
         // signed license
@@ -133,7 +129,7 @@ public class LicensesMetaDataSerializationTests extends ESTestCase {
         builder.endArray();
         builder.endObject();
         builder.endObject();
-        licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(builder.bytes().toBytes());
+        licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(builder.bytes());
         assertThat(licensesMetaDataFromXContent.getLicense(), equalTo(signedLicense));
 
         // trial and signed license
@@ -143,14 +139,14 @@ public class LicensesMetaDataSerializationTests extends ESTestCase {
         builder.startArray("trial_licenses");
         contentBuilder = XContentFactory.contentBuilder(XContentType.JSON);
         trialLicense.toXContent(contentBuilder, new ToXContent.MapParams(Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true")));
-        builder.value(Base64.getEncoder().encodeToString(encrypt(contentBuilder.bytes().toBytes())));
+        builder.value(Base64.getEncoder().encodeToString(encrypt(BytesReference.toBytes(contentBuilder.bytes()))));
         builder.endArray();
         builder.startArray("signed_licenses");
         signedLicense.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endArray();
         builder.endObject();
         builder.endObject();
-        licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(builder.bytes().toBytes());
+        licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(builder.bytes());
         assertThat(licensesMetaDataFromXContent.getLicense(), equalTo(signedLicense));
 
         // license with later issue date is selected
@@ -162,7 +158,7 @@ public class LicensesMetaDataSerializationTests extends ESTestCase {
         builder.startArray("trial_licenses");
         contentBuilder = XContentFactory.contentBuilder(XContentType.JSON);
         trialLicense.toXContent(contentBuilder, new ToXContent.MapParams(Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true")));
-        builder.value(Base64.getEncoder().encodeToString(encrypt(contentBuilder.bytes().toBytes())));
+        builder.value(Base64.getEncoder().encodeToString(encrypt(BytesReference.toBytes(contentBuilder.bytes()))));
         builder.endArray();
         builder.startArray("signed_licenses");
         signedLicense.toXContent(builder, ToXContent.EMPTY_PARAMS);
@@ -170,7 +166,7 @@ public class LicensesMetaDataSerializationTests extends ESTestCase {
         builder.endArray();
         builder.endObject();
         builder.endObject();
-        licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(builder.bytes().toBytes());
+        licensesMetaDataFromXContent = getLicensesMetaDataFromXContent(builder.bytes());
         assertThat(licensesMetaDataFromXContent.getLicense(), equalTo(signedLicenseIssuedLater));
 
     }
@@ -190,13 +186,12 @@ public class LicensesMetaDataSerializationTests extends ESTestCase {
         output.writeVInt(1);
         XContentBuilder contentBuilder = XContentFactory.contentBuilder(XContentType.JSON);
         trialLicense.toXContent(contentBuilder, new ToXContent.MapParams(Collections.singletonMap(License.LICENSE_SPEC_VIEW_MODE, "true")));
-        output.writeString(Base64.getEncoder().encodeToString(encrypt(contentBuilder.bytes().toBytes())));
+        output.writeString(Base64.getEncoder().encodeToString(encrypt(BytesReference.toBytes(contentBuilder.bytes()))));
-        byte[] bytes = output.bytes().toBytes();
+        try (StreamInput input = output.bytes().streamInput()) {
-        ByteBufferStreamInput input = new ByteBufferStreamInput(ByteBuffer.wrap(bytes));
-
             input.setVersion(Version.V_2_0_0_beta1);
             LicensesMetaData licensesMetaData = LicensesMetaData.PROTO.readFrom(input);
             assertThat(licensesMetaData.getLicense(), equalTo(trialLicense));
+        }
 
         // signed license
         License signedLicense = TestUtils.generateSignedLicense(TimeValue.timeValueHours(2));
@@ -204,23 +199,23 @@ public class LicensesMetaDataSerializationTests extends ESTestCase {
         output.writeVInt(1);
         signedLicense.writeTo(output);
         output.writeVInt(0);
-        bytes = output.bytes().toBytes();
+        try (StreamInput input = output.bytes().streamInput()) {
-        input = new ByteBufferStreamInput(ByteBuffer.wrap(bytes));
             input.setVersion(Version.V_2_0_0_beta1);
-        licensesMetaData = LicensesMetaData.PROTO.readFrom(input);
+            LicensesMetaData licensesMetaData = LicensesMetaData.PROTO.readFrom(input);
             assertThat(licensesMetaData.getLicense(), equalTo(signedLicense));
         }
+    }
 
     public void testLicenseTombstoneFromXContext() throws Exception {
         final XContentBuilder builder = XContentFactory.jsonBuilder();
         builder.startObject("licenses");
         builder.nullField("license");
         builder.endObject();
-        LicensesMetaData metaDataFromXContent = getLicensesMetaDataFromXContent(builder.bytes().toBytes());
+        LicensesMetaData metaDataFromXContent = getLicensesMetaDataFromXContent(builder.bytes());
         assertThat(metaDataFromXContent.getLicense(), equalTo(LicensesMetaData.LICENSE_TOMBSTONE));
     }
 
-    private static LicensesMetaData getLicensesMetaDataFromXContent(byte[] bytes) throws Exception {
+    private static LicensesMetaData getLicensesMetaDataFromXContent(BytesReference bytes) throws Exception {
         final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(bytes);
         parser.nextToken(); // consume null
         parser.nextToken(); // consume "licenses"

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringFeatureSet.java

@@ -82,7 +82,7 @@ public class MonitoringFeatureSet implements XPackFeatureSet {
 
         private static final String ENABLED_EXPORTERS_XFIELD = "enabled_exporters";
 
-        private @Nullable Map<String, Object> exporters;
+        @Nullable private Map<String, Object> exporters;
 
         public Usage(StreamInput in) throws IOException {
             super(in);

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/agent/AgentService.java

@@ -41,7 +41,7 @@ import java.util.concurrent.locks.ReentrantLock;
  * @see #stopCollection()
  * @see #startCollection()
  */
-public class AgentService extends AbstractLifecycleComponent<AgentService> {
+public class AgentService extends AbstractLifecycleComponent {
 
     private volatile ExportingWorker exportingWorker;
 

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/agent/collector/AbstractCollector.java

@@ -19,7 +19,7 @@ import org.elasticsearch.xpack.monitoring.agent.exporter.MonitoringDoc;
 
 import java.util.Collection;
 
-public abstract class AbstractCollector<T> extends AbstractLifecycleComponent<T> implements Collector<T> {
+public abstract class AbstractCollector extends AbstractLifecycleComponent implements Collector {
 
     private final String name;
 
@@ -48,9 +48,9 @@ public abstract class AbstractCollector<T> extends AbstractLifecycleComponent<T>
     }
 
     @Override
-    public T start() {
+    public void start() {
         logger.debug("starting collector [{}]", name());
-        return super.start();
+        super.start();
     }
 
     @Override
@@ -90,9 +90,9 @@ public abstract class AbstractCollector<T> extends AbstractLifecycleComponent<T>
     protected abstract Collection<MonitoringDoc> doCollect() throws Exception;
 
     @Override
-    public T stop() {
+    public void stop() {
         logger.debug("stopping collector [{}]", name());
-        return super.stop();
+        super.stop();
     }
 
     @Override

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/agent/collector/Collector.java

@@ -10,7 +10,7 @@ import org.elasticsearch.xpack.monitoring.agent.exporter.MonitoringDoc;
 
 import java.util.Collection;
 
-public interface Collector<T> extends LifecycleComponent<T> {
+public interface Collector extends LifecycleComponent {
 
     String name();
 

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/agent/collector/cluster/ClusterStateCollector.java

@@ -30,7 +30,7 @@ import java.util.List;
  * This collector runs on the master node only and collects {@link ClusterStateMonitoringDoc} document
  * at a given frequency.
  */
-public class ClusterStateCollector extends AbstractCollector<ClusterStateCollector> {
+public class ClusterStateCollector extends AbstractCollector {
 
     public static final String NAME = "cluster-state-collector";
 

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/agent/collector/cluster/ClusterStatsCollector.java

@@ -36,7 +36,7 @@ import java.util.List;
  * document; the cluster stats are also indexed in the timestamped index in a
  * "cluster_stats" document.
  */
-public class ClusterStatsCollector extends AbstractCollector<ClusterStatsCollector> {
+public class ClusterStatsCollector extends AbstractCollector {
 
     public static final String NAME = "cluster-stats-collector";
 

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/agent/collector/indices/IndexRecoveryCollector.java

@@ -32,7 +32,7 @@ import java.util.List;
  * This collector runs on the master node only and collects a {@link IndexRecoveryMonitoringDoc} document
  * for every index that has on-going shard recoveries.
  */
-public class IndexRecoveryCollector extends AbstractCollector<IndexRecoveryCollector> {
+public class IndexRecoveryCollector extends AbstractCollector {
 
     public static final String NAME = "index-recovery-collector";
 

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/agent/collector/indices/IndexStatsCollector.java

@@ -34,7 +34,7 @@ import java.util.List;
  * This collector runs on the master node only and collect a {@link IndexStatsMonitoringDoc} document
  * for each existing index in the cluster.
  */
-public class IndexStatsCollector extends AbstractCollector<IndexStatsCollector> {
+public class IndexStatsCollector extends AbstractCollector {
 
     public static final String NAME = "index-stats-collector";
 

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/agent/collector/indices/IndicesStatsCollector.java

@@ -29,7 +29,7 @@ import java.util.Collections;
  * <p>
  * This collector runs on the master node only and collect one {@link IndicesStatsMonitoringDoc} document.
  */
-public class IndicesStatsCollector extends AbstractCollector<IndicesStatsCollector> {
+public class IndicesStatsCollector extends AbstractCollector {
 
     public static final String NAME = "indices-stats-collector";
 

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/agent/collector/node/NodeStatsCollector.java

@@ -32,7 +32,7 @@ import java.util.Collections;
  * This collector runs on every non-client node and collect
  * a {@link NodeStatsMonitoringDoc} document for each node of the cluster.
  */
-public class NodeStatsCollector extends AbstractCollector<NodeStatsCollector> {
+public class NodeStatsCollector extends AbstractCollector {
 
     public static final String NAME = "node-stats-collector";
 

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/agent/collector/shards/ShardsCollector.java

@@ -30,7 +30,7 @@ import java.util.List;
  * This collector runs on the master node only and collects the {@link ShardMonitoringDoc} documents
  * for every index shard.
  */
-public class ShardsCollector extends AbstractCollector<ShardsCollector> {
+public class ShardsCollector extends AbstractCollector {
 
     public static final String NAME = "shards-collector";
 

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/agent/exporter/Exporter.java

@@ -23,7 +23,8 @@ public abstract class Exporter implements AutoCloseable {
     protected final Config config;
     protected final ESLogger logger;
 
-    protected final @Nullable TimeValue bulkTimeout;
+    @Nullable protected final TimeValue bulkTimeout;
+
     private AtomicBoolean closed = new AtomicBoolean(false);
 
     public Exporter(String type, Config config) {
@@ -103,7 +104,7 @@ public abstract class Exporter implements AutoCloseable {
         }
     }
 
-    public static abstract class Factory<E extends Exporter> {
+    public abstract static class Factory<E extends Exporter> {
 
         private final String type;
         private final boolean singleton;

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/agent/exporter/Exporters.java

@@ -32,7 +32,7 @@ import static java.util.Collections.emptyMap;
 /**
  *
  */
-public class Exporters extends AbstractLifecycleComponent<Exporters> implements Iterable<Exporter> {
+public class Exporters extends AbstractLifecycleComponent implements Iterable<Exporter> {
 
     private final Map<String, Exporter.Factory> factories;
     private final ClusterService clusterService;

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/agent/exporter/http/HttpExporter.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.monitoring.agent.exporter.http;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.SpecialPermission;
@@ -216,7 +217,8 @@ public class HttpExporter extends Exporter {
                 out.write(CONTENT_TYPE.xContent().streamSeparator());
 
                 // Render the monitoring document
-                out.write(resolver.source(doc, CONTENT_TYPE).toBytes());
+                BytesRef bytesRef = resolver.source(doc, CONTENT_TYPE).toBytesRef();
+                out.write(bytesRef.bytes, bytesRef.offset, bytesRef.length);
 
                 // Adds final bulk separator
                 out.write(CONTENT_TYPE.xContent().streamSeparator());
@@ -533,7 +535,7 @@ public class HttpExporter extends Exporter {
         }
     }
 
-    static private void validateHosts(String[] hosts) {
+    private static void validateHosts(String[] hosts) {
         for (String host : hosts) {
             try {
                 HttpExporterUtils.parseHostWithPath(host, "");
@@ -718,8 +720,9 @@ public class HttpExporter extends Exporter {
                         for (MonitoringDoc monitoringDoc : docs) {
                             try {
                                 render(monitoringDoc, buffer);
+                                BytesRef bytesRef = buffer.bytes().toBytesRef();
                                 // write the result to the connection
-                                out.write(buffer.bytes().toBytes());
+                                out.write(bytesRef.bytes, bytesRef.offset, bytesRef.length);
                             } finally {
                                 buffer.reset();
                             }

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/agent/resolver/MonitoringIndexNameResolver.java

@@ -126,7 +126,7 @@ public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
      * Data index name resolvers are used used to index documents in
      * the monitoring data index (.monitoring-data-{VERSION})
      */
-    public static abstract class Data<T extends MonitoringDoc> extends MonitoringIndexNameResolver<T> {
+    public abstract static class Data<T extends MonitoringDoc> extends MonitoringIndexNameResolver<T> {
 
         public static final String DATA = "data";
 
@@ -166,7 +166,7 @@ public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
      * Timestamped index name resolvers are used used to index documents in
      * a timestamped index (.monitoring-{ID}-{VERSION}-YYYY.MM.dd)
      */
-    public static abstract class Timestamped<T extends MonitoringDoc> extends MonitoringIndexNameResolver<T> {
+    public abstract static class Timestamped<T extends MonitoringDoc> extends MonitoringIndexNameResolver<T> {
 
         public static final Setting<String> INDEX_NAME_TIME_FORMAT_SETTING = new Setting<>("index.name.time_format", "YYYY.MM.dd",
                 Function.identity(), Setting.Property.NodeScope);

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/cleaner/CleanerService.java

@@ -26,7 +26,7 @@ import java.util.concurrent.ScheduledFuture;
 /**
  * {@code CleanerService} takes care of deleting old monitoring indices.
  */
-public class CleanerService extends AbstractLifecycleComponent<CleanerService> {
+public class CleanerService extends AbstractLifecycleComponent {
 
     private final MonitoringLicensee licensee;
     private final ThreadPool threadPool;

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/rest/MonitoringRestHandler.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.monitoring.rest;
 
-import org.elasticsearch.client.Client;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.rest.XPackRestHandler;
 
@@ -13,7 +12,7 @@ public abstract class MonitoringRestHandler extends XPackRestHandler {
 
     protected static String URI_BASE = XPackRestHandler.URI_BASE + "/monitoring";
 
-    public MonitoringRestHandler(Settings settings, Client client) {
+    public MonitoringRestHandler(Settings settings) {
-        super(settings, client);
+        super(settings);
     }
 }

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkAction.java

@@ -6,7 +6,7 @@
 package org.elasticsearch.xpack.monitoring.rest.action;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -32,8 +32,8 @@ public class RestMonitoringBulkAction extends MonitoringRestHandler {
     public static final String MONITORING_VERSION = "system_version";
 
     @Inject
-    public RestMonitoringBulkAction(Settings settings, RestController controller, Client client) {
+    public RestMonitoringBulkAction(Settings settings, RestController controller) {
-        super(settings, client);
+        super(settings);
         controller.registerHandler(POST, URI_BASE + "/_bulk", this);
         controller.registerHandler(PUT, URI_BASE + "/_bulk", this);
         controller.registerHandler(POST, URI_BASE + "/{type}/_bulk", this);
@@ -41,7 +41,7 @@ public class RestMonitoringBulkAction extends MonitoringRestHandler {
     }
 
     @Override
-    protected void handleRequest(RestRequest request, RestChannel channel, XPackClient client) throws Exception {
+    public void handleRequest(RestRequest request, RestChannel channel, XPackClient client) throws Exception {
         String defaultType = request.param("type");
 
         String id = request.param(MONITORING_ID);

elasticsearch/x-pack/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkDocTests.java

@@ -36,7 +36,7 @@ public class MonitoringBulkDocTests extends ESTestCase {
             output.setVersion(outputVersion);
             doc.writeTo(output);
 
-            StreamInput streamInput = StreamInput.wrap(output.bytes());
+            StreamInput streamInput = output.bytes().streamInput();
             streamInput.setVersion(randomVersion(random()));
             MonitoringBulkDoc doc2 = new MonitoringBulkDoc(streamInput);
 

elasticsearch/x-pack/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java

@@ -181,7 +181,7 @@ public class MonitoringBulkRequestTests extends ESTestCase {
         out.setVersion(randomVersion(random()));
         request.writeTo(out);
 
-        StreamInput in = StreamInput.wrap(out.bytes());
+        StreamInput in = out.bytes().streamInput();
         in.setVersion(out.getVersion());
         MonitoringBulkRequest request2 = new MonitoringBulkRequest();
         request2.readFrom(in);

elasticsearch/x-pack/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkResponseTests.java

@@ -57,7 +57,7 @@ public class MonitoringBulkResponseTests extends ESTestCase {
             output.setVersion(outputVersion);
             response.writeTo(output);
 
-            StreamInput streamInput = StreamInput.wrap(output.bytes());
+            StreamInput streamInput = output.bytes().streamInput();
             streamInput.setVersion(randomVersion(random()));
             MonitoringBulkResponse response2 = new MonitoringBulkResponse();
             response2.readFrom(streamInput);

elasticsearch/x-pack/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringIndexTests.java

@@ -56,7 +56,7 @@ public class MonitoringIndexTests extends ESTestCase {
 
         index.writeTo(out);
 
-        final StreamInput in = StreamInput.wrap(out.bytes().toBytes());
+        final StreamInput in = out.bytes().streamInput();
 
         assertSame(index, MonitoringIndex.readFrom(in));
 

elasticsearch/x-pack/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/agent/exporter/MonitoringDocTests.java

@@ -45,7 +45,7 @@ public class MonitoringDocTests extends ESTestCase {
             output.setVersion(outputVersion);
             monitoringDoc.writeTo(output);
 
-            StreamInput streamInput = StreamInput.wrap(output.bytes());
+            StreamInput streamInput = output.bytes().streamInput();
             streamInput.setVersion(randomVersion(random()));
             MonitoringDoc monitoringDoc2 = new MonitoringDoc(streamInput);
 

elasticsearch/x-pack/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/agent/exporter/http/HttpExporterTemplateTests.java

@@ -139,7 +139,7 @@ public class HttpExporterTemplateTests extends AbstractExporterTemplateTestCase
                         boolean templateExist = templates.containsKey(templateName);
 
                         if ("GET".equals(request.getMethod())) {
-                            return templateExist ? newResponse(200, templates.get(templateName).toUtf8()) : NOT_FOUND;
+                            return templateExist ? newResponse(200, templates.get(templateName).utf8ToString()) : NOT_FOUND;
                         }
                         if ("PUT".equals(request.getMethod())) {
                             templates.put(templateName, new BytesArray(request.getBody().readByteArray()));

elasticsearch/x-pack/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/agent/exporter/http/HttpExporterTests.java

@@ -467,7 +467,8 @@ public class HttpExporterTests extends MonitoringIntegTestCase {
 
     private void enqueueGetClusterVersionResponse(MockWebServer mockWebServer, Version v) throws IOException {
         mockWebServer.enqueue(new MockResponse().setResponseCode(200).setBody(
-                jsonBuilder().startObject().startObject("version").field("number", v.toString()).endObject().endObject().bytes().toUtf8()));
+                jsonBuilder().startObject().startObject("version").field("number", v.toString()).endObject().endObject().bytes()
+                        .utf8ToString()));
     }
 
     private void enqueueResponse(int responseCode, String body) throws IOException {

elasticsearch/x-pack/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/agent/resolver/cluster/ClusterStatsResolverTests.java

@@ -109,7 +109,7 @@ public class ClusterStatsResolverTests extends MonitoringIndexNameResolverTestCa
         BoundTransportAddress transportAddress = new BoundTransportAddress(new TransportAddress[]{DummyTransportAddress.INSTANCE},
                 DummyTransportAddress.INSTANCE);
         return new NodeInfo(Version.CURRENT, org.elasticsearch.Build.CURRENT,
-                new DiscoveryNode("node_0", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), emptyMap(),
+                new DiscoveryNode("node_0", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT),
                 Settings.EMPTY, DummyOsInfo.INSTANCE, new ProcessInfo(randomInt(), randomBoolean()), JvmInfo.jvmInfo(),
                 new ThreadPoolInfo(Collections.singletonList(new ThreadPool.Info("test_threadpool", ThreadPool.ThreadPoolType.FIXED, 5))),
                 new TransportInfo(transportAddress, Collections.emptyMap()), new HttpInfo(transportAddress, randomLong()),

elasticsearch/x-pack/security/bin/x-pack/migrate

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+SCRIPT="$0"
+
+# SCRIPT may be an arbitrarily deep series of symlinks. Loop until we have the concrete path.
+while [ -h "$SCRIPT" ] ; do
+  ls=`ls -ld "$SCRIPT"`
+  # Drop everything prior to ->
+  link=`expr "$ls" : '.*-> \(.*\)$'`
+  if expr "$link" : '/.*' > /dev/null; then
+    SCRIPT="$link"
+  else
+    SCRIPT=`dirname "$SCRIPT"`/"$link"
+  fi
+done
+
+# determine elasticsearch home
+ES_HOME=`dirname "$SCRIPT"`/../..
+
+# make ELASTICSEARCH_HOME absolute
+ES_HOME=`cd "$ES_HOME"; pwd`
+
+# If an include wasn't specified in the environment, then search for one...
+if [ "x$ES_INCLUDE" = "x" ]; then
+    # Locations (in order) to use when searching for an include file.
+    for include in /usr/share/elasticsearch/elasticsearch.in.sh \
+                   /usr/local/share/elasticsearch/elasticsearch.in.sh \
+                   /opt/elasticsearch/elasticsearch.in.sh \
+                   ~/.elasticsearch.in.sh \
+                   "`dirname "$0"`"/../elasticsearch.in.sh \
+                   "$ES_HOME/bin/elasticsearch.in.sh"; do
+        if [ -r "$include" ]; then
+            . "$include"
+            break
+        fi
+    done
+# ...otherwise, source the specified include.
+elif [ -r "$ES_INCLUDE" ]; then
+    . "$ES_INCLUDE"
+fi
+
+if [ -x "$JAVA_HOME/bin/java" ]; then
+    JAVA="$JAVA_HOME/bin/java"
+else
+    JAVA=`which java`
+fi
+
+if [ ! -x "$JAVA" ]; then
+    echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME"
+    exit 1
+fi
+
+if [ -z "$ES_CLASSPATH" ]; then
+    echo "You must set the ES_CLASSPATH var" >&2
+    exit 1
+fi
+
+# Try to read package config files
+if [ -f "/etc/sysconfig/elasticsearch" ]; then
+    CONF_DIR=/etc/elasticsearch
+
+    . "/etc/sysconfig/elasticsearch"
+elif [ -f "/etc/default/elasticsearch" ]; then
+    CONF_DIR=/etc/elasticsearch
+
+   . "/etc/default/elasticsearch"
+fi
+
+export HOSTNAME=`hostname -s`
+
+# include x-pack jars in classpath
+ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/plugins/x-pack/*"
+
+# don't let JAVA_TOOL_OPTIONS slip in (e.g. crazy agents in ubuntu)
+# works around https://bugs.launchpad.net/ubuntu/+source/jayatana/+bug/1441487
+if [ "x$JAVA_TOOL_OPTIONS" != "x" ]; then
+    echo "Warning: Ignoring JAVA_TOOL_OPTIONS=$JAVA_TOOL_OPTIONS"
+    echo "Please pass JVM parameters via ES_JAVA_OPTS instead"
+    unset JAVA_TOOL_OPTIONS
+fi
+
+# CONF_FILE setting was removed
+if [ ! -z "$CONF_FILE" ]; then
+    echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."
+    exit 1
+fi
+
+declare -a args=("$@")
+
+if [ -e "$CONF_DIR" ]; then
+  args=("${args[@]}" -Edefault.path.conf="$CONF_DIR")
+fi
+
+cd "$ES_HOME" > /dev/null
+"$JAVA" $ES_JAVA_OPTS -cp "$ES_CLASSPATH" -Des.path.home="$ES_HOME" org.elasticsearch.xpack.security.authc.esnative.ESNativeRealmMigrateTool "${args[@]}"
+status=$?
+cd - > /dev/null
+exit $status

elasticsearch/x-pack/security/bin/x-pack/migrate.bat

@@ -0,0 +1,9 @@
+@echo off
+
+rem Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+rem or more contributor license agreements. Licensed under the Elastic License;
+rem you may not use this file except in compliance with the Elastic License.
+
+PUSHD "%~dp0"
+CALL "%~dp0.in.bat" org.elasticsearch.xpack.security.authc.esnative.ESNativeRealmMigrateTool %*
+POPD

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatureSet.java

@@ -32,7 +32,7 @@ public class SecurityFeatureSet implements XPackFeatureSet {
 
     private final boolean enabled;
     private final SecurityLicenseState licenseState;
-    private final @Nullable Realms realms;
+    @Nullable private final Realms realms;
 
     @Inject
     public SecurityFeatureSet(Settings settings, @Nullable SecurityLicenseState licenseState,

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/action/user/ChangePasswordRequest.java

@@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.security.authc.support.CharArrays;
@@ -80,7 +81,7 @@ public class ChangePasswordRequest extends ActionRequest<ChangePasswordRequest>
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         username = in.readString();
-        passwordHash = CharArrays.utf8BytesToChars(in.readBytesReference().array());
+        passwordHash = CharArrays.utf8BytesToChars(BytesReference.toBytes(in.readBytesReference()));
         refreshPolicy = RefreshPolicy.readFrom(in);
     }
 

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/action/user/PutUserRequest.java

@@ -127,7 +127,7 @@ public class PutUserRequest extends ActionRequest<PutUserRequest> implements Use
         if (passwordHashRef == BytesArray.EMPTY) {
             passwordHash = null;
         } else {
-            passwordHash = CharArrays.utf8BytesToChars(passwordHashRef.array());
+            passwordHash = CharArrays.utf8BytesToChars(BytesReference.toBytes(passwordHashRef));
         }
         roles = in.readStringArray();
         fullName = in.readOptionalString();

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/action/user/PutUserRequestBuilder.java

@@ -79,6 +79,11 @@ public class PutUserRequestBuilder extends ActionRequestBuilder<PutUserRequest,
         return this;
     }
 
+    public PutUserRequestBuilder passwordHash(char[] passwordHash) {
+        request.passwordHash(passwordHash);
+        return this;
+    }
+
     public PutUserRequestBuilder source(String username, BytesReference source) throws IOException {
         username(username);
         try (XContentParser parser = XContentHelper.createParser(source)) {
@@ -99,6 +104,14 @@ public class PutUserRequestBuilder extends ActionRequestBuilder<PutUserRequest,
                         throw new ElasticsearchParseException(
                                 "expected field [{}] to be of type string, but found [{}] instead", currentFieldName, token);
                     }
+                } else if (ParseFieldMatcher.STRICT.match(currentFieldName, User.Fields.PASSWORD_HASH)) {
+                    if (token == XContentParser.Token.VALUE_STRING) {
+                        char[] passwordChars = parser.text().toCharArray();
+                        passwordHash(passwordChars);
+                    } else {
+                        throw new ElasticsearchParseException(
+                                "expected field [{}] to be of type string, but found [{}] instead", currentFieldName, token);
+                    }
                 } else if (ParseFieldMatcher.STRICT.match(currentFieldName, User.Fields.ROLES)) {
                     if (token == XContentParser.Token.VALUE_STRING) {
                         roles(Strings.commaDelimitedListToStringArray(parser.text()));

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditUtil.java

@@ -22,7 +22,7 @@ public class AuditUtil {
             try {
                 return XContentHelper.convertToJson(request.content(), false, false);
             } catch (IOException ioe) {
-                return "Invalid Format: " + request.content().toUtf8();
+                return "Invalid Format: " + request.content().utf8ToString();
             }
         }
         return "";

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java

@@ -44,7 +44,7 @@ import static org.elasticsearch.xpack.security.Security.setting;
 /**
  *
  */
-public class LoggingAuditTrail extends AbstractLifecycleComponent<LoggingAuditTrail> implements AuditTrail {
+public class LoggingAuditTrail extends AbstractLifecycleComponent implements AuditTrail {
 
     public static final String NAME = "logfile";
     public static final Setting<Boolean> HOST_ADDRESS_SETTING =

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authc/Authentication.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.security.authc;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -132,7 +133,7 @@ public class Authentication {
         BytesStreamOutput output = new BytesStreamOutput();
         Version.writeVersion(Version.CURRENT, output);
         writeTo(output);
-        return Base64.getEncoder().encodeToString(output.bytes().toBytes());
+        return Base64.getEncoder().encodeToString(BytesReference.toBytes(output.bytes()));
     }
 
     void writeTo(StreamOutput out) throws IOException {

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authc/Realm.java

@@ -111,7 +111,7 @@ public abstract class Realm<T extends AuthenticationToken> implements Comparable
      * settings. The factory will be called when creating a realm during the parsing of realms defined in the
      * elasticsearch.yml file
      */
-    public static abstract class Factory<R extends Realm> {
+    public abstract static class Factory<R extends Realm> {
 
         private final String type;
         private final boolean internal;

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java

@@ -32,7 +32,7 @@ import static org.elasticsearch.xpack.security.Security.setting;
 /**
  * Serves as a realms registry (also responsible for ordering the realms appropriately)
  */
-public class Realms extends AbstractLifecycleComponent<Realms> implements Iterable<Realm> {
+public class Realms extends AbstractLifecycleComponent implements Iterable<Realm> {
 
     public static final Setting<Settings> REALMS_GROUPS_SETTINGS = Setting.groupSetting(setting("authc.realms."), Property.NodeScope);
 

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authc/activedirectory/ActiveDirectoryGroupsResolver.java

@@ -88,7 +88,7 @@ public class ActiveDirectoryGroupsResolver implements GroupsResolver {
      *
      * @param SID byte encoded security ID
      */
-    static public String binarySidToStringSid(byte[] SID) {
+    public static String binarySidToStringSid(byte[] SID) {
         String strSID;
 
         //convert the SID into string format

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateTool.java

@@ -0,0 +1,349 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.authc.esnative;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Joiner;
+import javax.net.ssl.HttpsURLConnection;
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.cli.MultiCommand;
+import org.elasticsearch.cli.SettingCommand;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.internal.InternalSettingsPreparer;
+import org.elasticsearch.xpack.security.action.role.PutRoleRequest;
+import org.elasticsearch.xpack.security.action.user.PutUserRequest;
+import org.elasticsearch.xpack.security.authc.Realms;
+import org.elasticsearch.xpack.security.authc.file.FileUserPasswdStore;
+import org.elasticsearch.xpack.security.authc.file.FileUserRolesStore;
+import org.elasticsearch.xpack.security.authc.support.Hasher;
+import org.elasticsearch.xpack.security.authc.support.SecuredString;
+import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
+import org.elasticsearch.xpack.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.security.authz.permission.Permission;
+import org.elasticsearch.xpack.security.authz.store.FileRolesStore;
+import org.elasticsearch.xpack.security.ssl.ClientSSLService;
+import org.elasticsearch.xpack.security.ssl.SSLConfiguration;
+import org.elasticsearch.xpack.security.support.NoOpLogger;
+import org.elasticsearch.xpack.security.support.Validation;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.xpack.security.Security.setting;
+
+/**
+ * This is the command-line tool used for migrating users and roles from the file-based realm into the new native realm using the API for
+ * import. It reads from the files and tries its best to add the users, showing an error if it was incapable of importing them. Any existing
+ * users or roles are skipped.
+ */
+public class ESNativeRealmMigrateTool extends MultiCommand {
+
+    public static void main(String[] args) throws Exception {
+        exit(new ESNativeRealmMigrateTool().main(args, Terminal.DEFAULT));
+    }
+
+    public ESNativeRealmMigrateTool() {
+        super("Imports file-based users and roles to the native security realm");
+        subcommands.put("native", new MigrateUserOrRoles());
+    }
+
+    /** Command to migrate users and roles to the native realm */
+    public static class MigrateUserOrRoles extends SettingCommand {
+
+        private final OptionSpec<String> username;
+        private final OptionSpec<String> password;
+        private final OptionSpec<String> url;
+        private final OptionSpec<String> usersToMigrateCsv;
+        private final OptionSpec<String> rolesToMigrateCsv;
+        private final OptionSpec<String> esConfigDir;
+
+        public MigrateUserOrRoles() {
+            super("Migrates users or roles from file to native realm");
+            this.username = parser.acceptsAll(Arrays.asList("u", "username"),
+                    "User used to authenticate with Elasticsearch")
+                    .withRequiredArg();
+            this.password = parser.acceptsAll(Arrays.asList("p", "password"),
+                    "Password used to authenticate with Elasticsearch")
+                    .withRequiredArg();
+            this.url = parser.acceptsAll(Arrays.asList("U", "url"),
+                    "URL of Elasticsearch host")
+                    .withRequiredArg();
+            this.usersToMigrateCsv = parser.acceptsAll(Arrays.asList("n", "users"),
+                    "Users to migrate from file to native realm")
+                    .withRequiredArg();
+            this.rolesToMigrateCsv = parser.acceptsAll(Arrays.asList("r", "roles"),
+                    "Roles to migrate from file to native realm")
+                    .withRequiredArg();
+            this.esConfigDir = parser.acceptsAll(Arrays.asList("c", "config"),
+                    "Configuration directory to use instead of default")
+                    .withRequiredArg();
+        }
+
+        // Visible for testing
+        public OptionParser getParser() {
+            return this.parser;
+        }
+
+        @Override
+        protected void printAdditionalHelp(Terminal terminal) {
+            terminal.println("This tool migrates file based users[1] and roles[2] to the native realm in");
+            terminal.println("elasticsearch, saving the administrator from needing to manually transition");
+            terminal.println("them from the file.");
+        }
+
+        // Visible for testing
+        @Override
+        public void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
+            terminal.println("starting migration of users and roles...");
+            Settings.Builder sb = Settings.builder();
+            sb.put(settings);
+            if (this.esConfigDir != null) {
+                sb.put("path.conf", this.esConfigDir.value(options));
+            }
+            Settings shieldSettings = sb.build();
+            Environment shieldEnv = new Environment(shieldSettings);
+            importUsers(terminal, shieldSettings, shieldEnv, options);
+            importRoles(terminal, shieldSettings, shieldEnv, options);
+            terminal.println("users and roles imported.");
+        }
+
+        private String postURL(Settings settings, Environment env, String method, String urlString,
+                               OptionSet options, @Nullable String bodyString) throws Exception {
+            URI uri = new URI(urlString);
+            URL url = uri.toURL();
+            HttpURLConnection conn;
+            // If using SSL, need a custom service because it's likely a self-signed certificate
+            if ("https".equalsIgnoreCase(uri.getScheme())) {
+                Settings sslSettings = settings.getByPrefix(setting("http.ssl."));
+                SSLConfiguration.Global globalConfig = new SSLConfiguration.Global(settings);
+                final ClientSSLService sslService = new ClientSSLService(sslSettings, globalConfig);
+                sslService.setEnvironment(env);
+                final HttpsURLConnection httpsConn = (HttpsURLConnection) url.openConnection();
+                AccessController.doPrivileged(new PrivilegedAction<Void>() {
+                    @Override
+                    public Void run() {
+                        // Requires permission java.lang.RuntimePermission "setFactory";
+                        httpsConn.setSSLSocketFactory(sslService.sslSocketFactory(sslSettings));
+                        return null;
+                    }
+                });
+                conn = httpsConn;
+            } else {
+                conn = (HttpURLConnection) url.openConnection();
+            }
+            conn.setRequestMethod(method);
+            conn.setReadTimeout(30 * 1000); // 30 second timeout
+            // Add basic-auth header
+            conn.setRequestProperty("Authorization",
+                    UsernamePasswordToken.basicAuthHeaderValue(username.value(options),
+                            new SecuredString(password.value(options).toCharArray())));
+            conn.setDoOutput(true); // we'll be sending a body
+            conn.connect();
+            if (bodyString != null) {
+                try (OutputStream out = conn.getOutputStream()) {
+                    out.write(bodyString.getBytes(Charsets.UTF_8));
+                } catch (Exception e) {
+                    try {
+                        conn.disconnect();
+                    } catch (Exception e2) {
+                        // Ignore exceptions if we weren't able to close the connection after an error
+                    }
+                    throw e;
+                }
+            }
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream(), Charsets.UTF_8))) {
+                StringBuilder sb = new StringBuilder();
+                String line = null;
+                while ((line = reader.readLine()) != null) {
+                    sb.append(line);
+                }
+                return sb.toString();
+            } catch (IOException e) {
+                try (BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getErrorStream(), Charsets.UTF_8))) {
+                    StringBuilder sb = new StringBuilder();
+                    String line = null;
+                    while ((line = reader.readLine()) != null) {
+                        sb.append(line);
+                    }
+                    throw new IOException(sb.toString(), e);
+                }
+            } finally {
+                conn.disconnect();
+            }
+        }
+
+        public Set<String> getUsersThatExist(Terminal terminal, Settings settings, Environment env, OptionSet options) throws Exception {
+            Set<String> existingUsers = new HashSet<>();
+            String allUsersJson = postURL(settings, env, "GET", this.url.value(options) + "/_xpack/security/user/", options, null);
+            try (XContentParser parser = JsonXContent.jsonXContent.createParser(allUsersJson)) {
+                XContentParser.Token token = parser.nextToken();
+                String userName;
+                if (token == XContentParser.Token.START_OBJECT) {
+                    while ((token = parser.nextToken()) == XContentParser.Token.FIELD_NAME) {
+                        userName = parser.currentName();
+                        existingUsers.add(userName);
+                        parser.nextToken();
+                        parser.skipChildren();
+                    }
+                } else {
+                    throw new ElasticsearchException("failed to retrieve users, expecting an object but got: " + token);
+                }
+            }
+            terminal.println("found existing users: " + existingUsers);
+            return existingUsers;
+        }
+
+        public static String createUserJson(String[] roles, char[] password) throws IOException {
+            XContentBuilder builder = jsonBuilder();
+            builder.startObject();
+            {
+                builder.field("password_hash", new String(password));
+                builder.startArray("roles");
+                for (String role : roles) {
+                    builder.value(role);
+                }
+                builder.endArray();
+            }
+            builder.endObject();
+            return builder.string();
+        }
+
+        public void importUsers(Terminal terminal, Settings settings, Environment env, OptionSet options) {
+            String usersCsv = usersToMigrateCsv.value(options);
+            String[] usersToMigrate = (usersCsv != null) ? usersCsv.split(",") : Strings.EMPTY_ARRAY;
+            Settings fileRealmSettings = Realms.fileRealmSettings(settings);
+            Path usersFile = FileUserPasswdStore.resolveFile(fileRealmSettings, env);
+            Path usersRolesFile = FileUserRolesStore.resolveFile(fileRealmSettings, env);
+            terminal.println("importing users from [" + usersFile + "]...");
+            Map<String, char[]> userToHashedPW = FileUserPasswdStore.parseFile(usersFile, null);
+            Map<String, String[]> userToRoles = FileUserRolesStore.parseFile(usersRolesFile, null);
+            Set<String> existingUsers;
+            try {
+                existingUsers = getUsersThatExist(terminal, settings, env, options);
+            } catch (Exception e) {
+                throw new ElasticsearchException("failed to get users that already exist, skipping user import", e);
+            }
+            if (usersToMigrate.length == 0) {
+                usersToMigrate = userToHashedPW.keySet().toArray(new String[userToHashedPW.size()]);
+            }
+            for (String user : usersToMigrate) {
+                if (userToHashedPW.containsKey(user) == false) {
+                    terminal.println("no user [" + user + "] found, skipping");
+                    continue;
+                } else if (existingUsers.contains(user)) {
+                    terminal.println("user [" + user + "] already exists, skipping");
+                    continue;
+                }
+                terminal.println("migrating user [" + user + "]");
+                String reqBody = "n/a";
+                try {
+                    reqBody = createUserJson(userToRoles.get(user), userToHashedPW.get(user));
+                    String resp = postURL(settings, env, "POST",
+                            this.url.value(options) + "/_xpack/security/user/" + user, options, reqBody);
+                    terminal.println(resp);
+                } catch (Exception e) {
+                    throw new ElasticsearchException("failed to migrate user [" + user + "] with body: " + reqBody, e);
+                }
+            }
+        }
+
+        public Set<String> getRolesThatExist(Terminal terminal, Settings settings, Environment env, OptionSet options) throws Exception {
+            Set<String> existingRoles = new HashSet<>();
+            String allRolesJson = postURL(settings, env, "GET", this.url.value(options) + "/_xpack/security/role/", options, null);
+            try (XContentParser parser = JsonXContent.jsonXContent.createParser(allRolesJson)) {
+                XContentParser.Token token = parser.nextToken();
+                String roleName;
+                if (token == XContentParser.Token.START_OBJECT) {
+                    while ((token = parser.nextToken()) == XContentParser.Token.FIELD_NAME) {
+                        roleName = parser.currentName();
+                        existingRoles.add(roleName);
+                        parser.nextToken();
+                        parser.skipChildren();
+                    }
+                } else {
+                    throw new ElasticsearchException("failed to retrieve roles, expecting an object but got: " + token);
+                }
+            }
+            terminal.println("found existing roles: " + existingRoles);
+            return existingRoles;
+        }
+
+        public static String createRoleJson(RoleDescriptor rd) throws IOException {
+            XContentBuilder builder = jsonBuilder();
+            rd.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            return builder.string();
+        }
+
+        public void importRoles(Terminal terminal, Settings settings, Environment env, OptionSet options) {
+            String rolesCsv = rolesToMigrateCsv.value(options);
+            String[] rolesToMigrate = (rolesCsv != null) ? rolesCsv.split(",") : Strings.EMPTY_ARRAY;
+            Settings fileRealmSettings = Realms.fileRealmSettings(settings);
+            Path rolesFile = FileRolesStore.resolveFile(fileRealmSettings, env).toAbsolutePath();
+            terminal.println("importing roles from [" + rolesFile + "]...");
+            Map<String, RoleDescriptor> roles = FileRolesStore.parseRoleDescriptors(rolesFile, null, true, Settings.EMPTY);
+            Set<String> existingRoles;
+            try {
+                existingRoles = getRolesThatExist(terminal, settings, env, options);
+            } catch (Exception e) {
+                throw new ElasticsearchException("failed to get roles that already exist, skipping role import", e);
+            }
+            if (rolesToMigrate.length == 0) {
+                rolesToMigrate = roles.keySet().toArray(new String[roles.size()]);
+            }
+            for (String roleName : rolesToMigrate) {
+                if (roles.containsKey(roleName) == false) {
+                    terminal.println("no role [" + roleName + "] found, skipping");
+                    continue;
+                } else if (existingRoles.contains(roleName)) {
+                    terminal.println("role [" + roleName + "] already exists, skipping");
+                    continue;
+                }
+                terminal.println("migrating role [" + roleName + "]");
+                String reqBody = "n/a";
+                try {
+                    reqBody = createRoleJson(roles.get(roleName));;
+                    String resp = postURL(settings, env, "POST",
+                            this.url.value(options) + "/_xpack/security/role/" + roleName, options, reqBody);
+                    terminal.println(resp);
+                } catch (Exception e) {
+                    throw new ElasticsearchException("failed to migrate role [" + roleName + "] with body: " + reqBody, e);
+                }
+            }
+        }
+    }
+}

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/AbstractLdapRealm.java

@@ -101,7 +101,7 @@ public abstract class AbstractLdapRealm extends CachingUsernamePasswordRealm {
         }
     }
 
-    public static abstract class Factory<R extends AbstractLdapRealm> extends UsernamePasswordRealm.Factory<R> {
+    public abstract static class Factory<R extends AbstractLdapRealm> extends UsernamePasswordRealm.Factory<R> {
 
         public Factory(String type, RestController restController) {
             super(type, restController, false);

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authc/support/BCrypt.java

@@ -337,13 +337,13 @@ public class BCrypt {
     // bcrypt IV: "OrpheanBeholderScryDoubt". The C implementation calls
     // this "ciphertext", but it is really plaintext or an IV. We keep
     // the name to make code comparison easier.
-    static private final int bf_crypt_ciphertext[] = {
+    private static final int bf_crypt_ciphertext[] = {
         0x4f727068, 0x65616e42, 0x65686f6c,
         0x64657253, 0x63727944, 0x6f756274
     };
 
     // Table for Base64 encoding
-    static private final char base64_code[] = {
+    private static final char base64_code[] = {
         '.', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
         'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
         'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h',
@@ -353,7 +353,7 @@ public class BCrypt {
     };
 
     // Table for Base64 decoding
-    static private final byte index_64[] = {
+    private static final byte index_64[] = {
         -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
         -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
         -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authc/support/UsernamePasswordRealm.java

@@ -31,7 +31,7 @@ public abstract class UsernamePasswordRealm extends Realm<UsernamePasswordToken>
         return token instanceof UsernamePasswordToken;
     }
 
-    public static abstract class Factory<R extends UsernamePasswordRealm> extends Realm.Factory<R> {
+    public abstract static class Factory<R extends UsernamePasswordRealm> extends Realm.Factory<R> {
 
         protected Factory(String type, RestController restController, boolean internal) {
             super(type, internal);

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/RoleDescriptor.java

@@ -17,7 +17,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -303,7 +302,7 @@ public class RoleDescriptor implements ToXContent {
             sb.append("], privileges=[").append(Strings.arrayToCommaDelimitedString(privileges));
             sb.append("], fields=[").append(Strings.arrayToCommaDelimitedString(fields));
             if (query != null) {
-                sb.append("], query=").append(query.toUtf8());
+                sb.append("], query=").append(query.utf8ToString());
             }
             sb.append("]");
             return sb.toString();
@@ -340,7 +339,7 @@ public class RoleDescriptor implements ToXContent {
                 builder.array("fields", fields);
             }
             if (query != null) {
-                builder.field("query", query.toUtf8());
+                builder.field("query", query.utf8ToString());
             }
             return builder.endObject();
         }
@@ -368,7 +367,7 @@ public class RoleDescriptor implements ToXContent {
             out.writeStringArray(privileges);
             if (query != null) {
                 out.writeBoolean(true);
-                out.writeByteArray(query.array());
+                out.writeByteArray(BytesReference.toBytes(query));
             } else {
                 out.writeBoolean(false);
             }

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/DocumentSubsetReader.java

@@ -31,7 +31,7 @@ public final class DocumentSubsetReader extends FilterLeafReader {
         return new DocumentSubsetDirectoryReader(in, bitsetFilterCache, roleQuery);
     }
 
-    final static class DocumentSubsetDirectoryReader extends FilterDirectoryReader {
+    static final class DocumentSubsetDirectoryReader extends FilterDirectoryReader {
 
         private final Query roleQuery;
         private final BitsetFilterCache bitsetFilterCache;

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldSubsetReader.java

@@ -156,7 +156,7 @@ public final class FieldSubsetReader extends FilterLeafReader {
                     Tuple<XContentType, Map<String, Object>> result = XContentHelper.convertToMap(bytes, true);
                     Map<String, Object> transformedSource = XContentMapValues.filter(result.v2(), fieldNames, null);
                     XContentBuilder xContentBuilder = XContentBuilder.builder(result.v1().xContent()).map(transformedSource);
-                    visitor.binaryField(fieldInfo, xContentBuilder.bytes().toBytes());
+                    visitor.binaryField(fieldInfo, BytesReference.toBytes(xContentBuilder.bytes()));
                 } else {
                     visitor.binaryField(fieldInfo, value);
                 }

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/privilege/ClusterPrivilege.java

@@ -40,7 +40,7 @@ public class ClusterPrivilege extends AbstractAutomatonPrivilege<ClusterPrivileg
     public static final ClusterPrivilege TRANSPORT_CLIENT =      new ClusterPrivilege("transport_client",       TRANSPORT_CLIENT_AUTOMATON);
     public static final ClusterPrivilege MANAGE_SECURITY =       new ClusterPrivilege("manage_security",        MANAGE_SECURITY_AUTOMATON);
 
-    public final static Predicate<String> ACTION_MATCHER = ClusterPrivilege.ALL.predicate();
+    public static final Predicate<String> ACTION_MATCHER = ClusterPrivilege.ALL.predicate();
 
     private static final Set<ClusterPrivilege> values = new CopyOnWriteArraySet<>();
 

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java

@@ -8,6 +8,7 @@ package org.elasticsearch.xpack.security.authz.store;
 import com.fasterxml.jackson.dataformat.yaml.snakeyaml.error.YAMLException;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
@@ -48,7 +49,7 @@ import static org.elasticsearch.xpack.security.Security.setting;
 /**
  *
  */
-public class FileRolesStore extends AbstractLifecycleComponent<RolesStore> implements RolesStore {
+public class FileRolesStore extends AbstractLifecycleComponent implements RolesStore {
 
     public static final Setting<String> ROLES_FILE_SETTING =
             Setting.simpleString(setting("authz.store.files.roles"), Property.NodeScope);
@@ -150,7 +151,55 @@ public class FileRolesStore extends AbstractLifecycleComponent<RolesStore> imple
         return unmodifiableMap(roles);
     }
 
+    public static Map<String, RoleDescriptor> parseRoleDescriptors(Path path, ESLogger logger,
+                                                                   boolean resolvePermission, Settings settings) {
+        if (logger == null) {
+            logger = NoOpLogger.INSTANCE;
+        }
+
+        Map<String, RoleDescriptor> roles = new HashMap<>();
+        logger.trace("attempted to read roles file located at [{}]", path.toAbsolutePath());
+        if (Files.exists(path)) {
+            try {
+                List<String> roleSegments = roleSegments(path);
+                for (String segment : roleSegments) {
+                    RoleDescriptor rd = parseRoleDescriptor(segment, path, logger, resolvePermission, settings);
+                    if (rd != null) {
+                        roles.put(rd.getName(), rd);
+                    }
+                }
+            } catch (IOException ioe) {
+                logger.error("failed to read roles file [{}]. skipping all roles...", ioe, path.toAbsolutePath());
+            }
+        }
+        return unmodifiableMap(roles);
+    }
+
+    @Nullable
     private static Role parseRole(String segment, Path path, ESLogger logger, boolean resolvePermissions, Settings settings) {
+        RoleDescriptor descriptor = parseRoleDescriptor(segment, path, logger, resolvePermissions, settings);
+
+        if (descriptor != null) {
+            String roleName = descriptor.getName();
+            // first check if FLS/DLS is enabled on the role...
+            for (RoleDescriptor.IndicesPrivileges privilege : descriptor.getIndicesPrivileges()) {
+                if ((privilege.getQuery() != null || privilege.getFields() != null)
+                        && Security.flsDlsEnabled(settings) == false) {
+                    logger.error("invalid role definition [{}] in roles file [{}]. document and field level security is not " +
+                                    "enabled. set [{}] to [true] in the configuration file. skipping role...", roleName, path
+                            .toAbsolutePath(), XPackPlugin.featureEnabledSetting(Security.DLS_FLS_FEATURE));
+                    return null;
+                }
+            }
+            return Role.builder(descriptor).build();
+        } else {
+            return null;
+        }
+    }
+
+    @Nullable
+    private static RoleDescriptor parseRoleDescriptor(String segment, Path path, ESLogger logger,
+                                                      boolean resolvePermissions, Settings settings) {
         String roleName = null;
         try {
             XContentParser parser = YamlXContent.yamlXContent.createParser(segment);
@@ -167,25 +216,13 @@ public class FileRolesStore extends AbstractLifecycleComponent<RolesStore> imple
                     }
 
                     if (resolvePermissions == false) {
-                        return Role.builder(roleName).build();
+                        return new RoleDescriptor(roleName, null, null, null);
                     }
 
                     token = parser.nextToken();
                     if (token == XContentParser.Token.START_OBJECT) {
                         RoleDescriptor descriptor = RoleDescriptor.parse(roleName, parser);
-
+                        return descriptor;
-                        // first check if FLS/DLS is enabled on the role...
-                        for (RoleDescriptor.IndicesPrivileges privilege : descriptor.getIndicesPrivileges()) {
-                            if ((privilege.getQuery() != null || privilege.getFields() != null)
-                                    && Security.flsDlsEnabled(settings) == false) {
-                                logger.error("invalid role definition [{}] in roles file [{}]. document and field level security is not " +
-                                        "enabled. set [{}] to [true] in the configuration file. skipping role...", roleName, path
-                                        .toAbsolutePath(), XPackPlugin.featureEnabledSetting(Security.DLS_FLS_FEATURE));
-                                return null;
-                            }
-                        }
-
-                        return Role.builder(descriptor).build();
                     } else {
                         logger.error("invalid role definition [{}] in roles file [{}]. skipping role...", roleName, path.toAbsolutePath());
                         return null;

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/crypto/InternalCryptoService.java

@@ -51,7 +51,7 @@ import static org.elasticsearch.xpack.security.Security.setting;
 /**
  *
  */
-public class InternalCryptoService extends AbstractLifecycleComponent<InternalCryptoService> implements CryptoService {
+public class InternalCryptoService extends AbstractLifecycleComponent implements CryptoService {
 
     public static final String KEY_ALGO = "HmacSHA512";
     public static final int KEY_SIZE = 1024;

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.security.rest;
 
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
@@ -56,7 +57,7 @@ public class SecurityRestFilter extends RestFilter {
     }
 
     @Override
-    public void process(RestRequest request, RestChannel channel, RestFilterChain filterChain) throws Exception {
+    public void process(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception {
 
         if (licenseState.authenticationAndAuthorizationEnabled()) {
             // CORS - allow for preflight unauthenticated OPTIONS request
@@ -70,7 +71,7 @@ public class SecurityRestFilter extends RestFilter {
             RemoteHostHeader.process(request, threadContext);
         }
 
-        filterChain.continueProcessing(request, channel);
+        filterChain.continueProcessing(request, channel, client);
     }
 
     static void putClientCertificateInContext(RestRequest request, ThreadContext threadContext, ESLogger logger) throws Exception {

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateAction.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.security.rest.action;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -31,14 +31,14 @@ public class RestAuthenticateAction extends BaseRestHandler {
     private final SecurityContext securityContext;
 
     @Inject
-    public RestAuthenticateAction(Settings settings, RestController controller, Client client, SecurityContext securityContext) {
+    public RestAuthenticateAction(Settings settings, RestController controller, SecurityContext securityContext) {
-        super(settings, client);
+        super(settings);
         this.securityContext = securityContext;
         controller.registerHandler(GET, "/_xpack/security/_authenticate", this);
     }
 
     @Override
-    protected void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
         final User user = securityContext.getUser();
         assert user != null;
         final String username = user.runAs() == null ? user.principal() : user.runAs().principal();

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/rest/action/realm/RestClearRealmCacheAction.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.security.rest.action.realm;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -21,14 +21,14 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 public class RestClearRealmCacheAction extends BaseRestHandler {
 
     @Inject
-    public RestClearRealmCacheAction(Settings settings, RestController controller, Client client) {
+    public RestClearRealmCacheAction(Settings settings, RestController controller) {
-        super(settings, client);
+        super(settings);
         controller.registerHandler(POST, "/_xpack/security/realm/{realms}/_cache/clear", this); // deprecated
         controller.registerHandler(POST, "/_xpack/security/realm/{realms}/_clear_cache", this);
     }
 
     @Override
-    protected void handleRequest(RestRequest request, final RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest request, final RestChannel channel, NodeClient client) throws Exception {
 
         String[] realms = request.paramAsStringArrayOrEmptyIfAll("realms");
         String[] usernames = request.paramAsStringArrayOrEmptyIfAll("usernames");

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestClearRolesCacheAction.java

@@ -5,7 +5,8 @@
  */
 package org.elasticsearch.xpack.security.rest.action.role;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -24,13 +25,13 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 public class RestClearRolesCacheAction extends BaseRestHandler {
 
     @Inject
-    public RestClearRolesCacheAction(Settings settings, RestController controller, Client client) {
+    public RestClearRolesCacheAction(Settings settings, RestController controller) {
-        super(settings, client);
+        super(settings);
         controller.registerHandler(POST, "/_xpack/security/role/{name}/_clear_cache", this);
     }
 
     @Override
-    protected void handleRequest(RestRequest request, final RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest request, final RestChannel channel, NodeClient client) throws Exception {
 
         String[] roles = request.paramAsStringArrayOrEmptyIfAll("name");
 

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestDeleteRoleAction.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.security.rest.action.role;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -27,13 +27,13 @@ import org.elasticsearch.xpack.security.client.SecurityClient;
 public class RestDeleteRoleAction extends BaseRestHandler {
 
     @Inject
-    public RestDeleteRoleAction(Settings settings, RestController controller, Client client) {
+    public RestDeleteRoleAction(Settings settings, RestController controller) {
-        super(settings, client);
+        super(settings);
         controller.registerHandler(RestRequest.Method.DELETE, "/_xpack/security/role/{name}", this);
     }
 
     @Override
-    protected void handleRequest(RestRequest request, final RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest request, final RestChannel channel, NodeClient client) throws Exception {
         DeleteRoleRequestBuilder requestBuilder = new SecurityClient(client).prepareDeleteRole(request.param("name"));
         if (request.hasParam("refresh")) {
             requestBuilder.refresh(request.paramAsBoolean("refresh", true));

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestGetRolesAction.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.security.rest.action.role;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -28,14 +28,14 @@ import org.elasticsearch.xpack.security.authz.RoleDescriptor;
 public class RestGetRolesAction extends BaseRestHandler {
 
     @Inject
-    public RestGetRolesAction(Settings settings, RestController controller, Client client) {
+    public RestGetRolesAction(Settings settings, RestController controller) {
-        super(settings, client);
+        super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_xpack/security/role/", this);
         controller.registerHandler(RestRequest.Method.GET, "/_xpack/security/role/{name}", this);
     }
 
     @Override
-    protected void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
         final String[] roles = request.paramAsStringArray("name", Strings.EMPTY_ARRAY);
         new SecurityClient(client).prepareGetRoles(roles).execute(new RestBuilderListener<GetRolesResponse>(channel) {
             @Override

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestPutRoleAction.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.security.rest.action.role;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -27,14 +27,14 @@ import org.elasticsearch.xpack.security.client.SecurityClient;
 public class RestPutRoleAction extends BaseRestHandler {
 
     @Inject
-    public RestPutRoleAction(Settings settings, RestController controller, Client client) {
+    public RestPutRoleAction(Settings settings, RestController controller) {
-        super(settings, client);
+        super(settings);
         controller.registerHandler(RestRequest.Method.POST, "/_xpack/security/role/{name}", this);
         controller.registerHandler(RestRequest.Method.PUT, "/_xpack/security/role/{name}", this);
     }
 
     @Override
-    protected void handleRequest(RestRequest request, final RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest request, final RestChannel channel, NodeClient client) throws Exception {
         PutRoleRequestBuilder requestBuilder = new SecurityClient(client).preparePutRole(request.param("name"), request.content());
         requestBuilder.setRefreshPolicy(request.param("refresh"));
         requestBuilder.execute(new RestBuilderListener<PutRoleResponse>(channel) {

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.security.rest.action.user;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -29,8 +29,8 @@ public class RestChangePasswordAction extends BaseRestHandler {
     private final SecurityContext securityContext;
 
     @Inject
-    public RestChangePasswordAction(Settings settings, Client client, RestController controller, SecurityContext securityContext) {
+    public RestChangePasswordAction(Settings settings, RestController controller, SecurityContext securityContext) {
-        super(settings, client);
+        super(settings);
         this.securityContext = securityContext;
         controller.registerHandler(RestRequest.Method.POST, "/_xpack/security/user/{username}/_password", this);
         controller.registerHandler(RestRequest.Method.PUT, "/_xpack/security/user/{username}/_password", this);
@@ -39,7 +39,7 @@ public class RestChangePasswordAction extends BaseRestHandler {
     }
 
     @Override
-    protected void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
         final User user = securityContext.getUser();
         String username = request.param("username");
         if (username == null) {

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestDeleteUserAction.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.security.rest.action.user;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -28,13 +28,13 @@ import org.elasticsearch.xpack.security.client.SecurityClient;
 public class RestDeleteUserAction extends BaseRestHandler {
 
     @Inject
-    public RestDeleteUserAction(Settings settings, RestController controller, Client client) {
+    public RestDeleteUserAction(Settings settings, RestController controller) {
-        super(settings, client);
+        super(settings);
         controller.registerHandler(RestRequest.Method.DELETE, "/_xpack/security/user/{username}", this);
     }
 
     @Override
-    protected void handleRequest(RestRequest request, final RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest request, final RestChannel channel, NodeClient client) throws Exception {
         String username = request.param("username");
 
         DeleteUserRequestBuilder requestBuilder = new SecurityClient(client).prepareDeleteUser(username);

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUsersAction.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.security.rest.action.user;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -29,14 +29,14 @@ import org.elasticsearch.xpack.security.client.SecurityClient;
 public class RestGetUsersAction extends BaseRestHandler {
 
     @Inject
-    public RestGetUsersAction(Settings settings, RestController controller, Client client) {
+    public RestGetUsersAction(Settings settings, RestController controller) {
-        super(settings, client);
+        super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_xpack/security/user/", this);
         controller.registerHandler(RestRequest.Method.GET, "/_xpack/security/user/{username}", this);
     }
 
     @Override
-    protected void handleRequest(RestRequest request, final RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest request, final RestChannel channel, NodeClient client) throws Exception {
         String[] usernames = request.paramAsStringArray("username", Strings.EMPTY_ARRAY);
 
         new SecurityClient(client).prepareGetUsers(usernames).execute(new RestBuilderListener<GetUsersResponse>(channel) {

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestPutUserAction.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.security.rest.action.user;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -27,14 +27,14 @@ import org.elasticsearch.xpack.security.client.SecurityClient;
 public class RestPutUserAction extends BaseRestHandler {
 
     @Inject
-    public RestPutUserAction(Settings settings, RestController controller, Client client) {
+    public RestPutUserAction(Settings settings, RestController controller) {
-        super(settings, client);
+        super(settings);
         controller.registerHandler(RestRequest.Method.POST, "/_xpack/security/user/{username}", this);
         controller.registerHandler(RestRequest.Method.PUT, "/_xpack/security/user/{username}", this);
     }
 
     @Override
-    protected void handleRequest(RestRequest request, final RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest request, final RestChannel channel, NodeClient client) throws Exception {
         PutUserRequestBuilder requestBuilder = new SecurityClient(client).preparePutUser(request.param("username"), request.content());
         if (request.hasParam("refresh")) {
             requestBuilder.setRefreshPolicy(request.param("refresh"));

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/support/AbstractSecurityModule.java

@@ -33,7 +33,7 @@ public abstract class AbstractSecurityModule extends AbstractModule {
 
     protected abstract void configure(boolean clientMode);
 
-    public static abstract class Node extends AbstractSecurityModule {
+    public abstract static class Node extends AbstractSecurityModule {
 
         protected Node(Settings settings) {
             super(settings);

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportService.java

@@ -211,7 +211,7 @@ public class SecurityServerTransportService extends TransportService {
      * This handler wrapper ensures that the response thread executes with the correct thread context. Before any of the4 handle methods
      * are invoked we restore the context.
      */
-    private final static class ContextRestoreResponseHandler<T extends TransportResponse> implements TransportResponseHandler<T> {
+    private static final class ContextRestoreResponseHandler<T extends TransportResponse> implements TransportResponseHandler<T> {
         private final TransportResponseHandler<T> delegate;
         private final ThreadContext.StoredContext threadContext;
 

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java

@@ -8,6 +8,7 @@ package org.elasticsearch.xpack.security.transport;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.transport.TcpTransportChannel;
 import org.elasticsearch.xpack.security.authc.Authentication;
 import org.elasticsearch.xpack.security.action.SecurityActionMapper;
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
@@ -16,7 +17,6 @@ import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.elasticsearch.transport.DelegatingTransportChannel;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.netty.NettyTransportChannel;
 import org.jboss.netty.channel.Channel;
 import org.jboss.netty.handler.ssl.SslHandler;
 
@@ -81,8 +81,9 @@ public interface ServerTransportFilter {
                 unwrappedChannel = ((DelegatingTransportChannel) unwrappedChannel).getChannel();
             }
 
-            if (extractClientCert && (unwrappedChannel instanceof NettyTransportChannel)) {
+            if (extractClientCert && (unwrappedChannel instanceof TcpTransportChannel)
-                Channel channel = ((NettyTransportChannel) unwrappedChannel).getChannel();
+                    && ((TcpTransportChannel) unwrappedChannel).getChannel() instanceof Channel) {
+                Channel channel = (Channel) ((TcpTransportChannel) unwrappedChannel).getChannel();
                 SslHandler sslHandler = channel.getPipeline().get(SslHandler.class);
                 assert sslHandler != null;
 

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/transport/netty/SecurityNettyTransport.java

@@ -21,6 +21,7 @@ import org.elasticsearch.xpack.security.transport.SSLClientAuth;
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.netty.NettyTransport;
+import org.jboss.netty.channel.Channel;
 import org.jboss.netty.channel.ChannelHandlerContext;
 import org.jboss.netty.channel.ChannelPipeline;
 import org.jboss.netty.channel.ChannelPipelineFactory;
@@ -32,6 +33,7 @@ import org.jboss.netty.handler.ssl.SslHandler;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLParameters;
 import java.net.InetSocketAddress;
+import java.util.Collections;
 import java.util.List;
 
 import static org.elasticsearch.xpack.security.Security.featureEnabledSetting;
@@ -77,7 +79,7 @@ public class SecurityNettyTransport extends NettyTransport {
 
     private final ServerSSLService serverSslService;
     private final ClientSSLService clientSSLService;
-    private final @Nullable IPFilter authenticator;
+    @Nullable private final IPFilter authenticator;
     private final boolean ssl;
 
     @Inject
@@ -111,30 +113,23 @@ public class SecurityNettyTransport extends NettyTransport {
     }
 
     @Override
-    protected void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
+    protected void onException(Channel channel, Throwable e) {
-        if (!lifecycle.started()) {
+        if (isNotSslRecordException(e)) {
-            return;
-        }
-
-        Throwable t = e.getCause();
-        if (isNotSslRecordException(t)) {
             if (logger.isTraceEnabled()) {
-                logger.trace("received plaintext traffic on a encrypted channel, closing connection {}", t, ctx.getChannel());
+                logger.trace("received plaintext traffic on a encrypted channel, closing connection {}", e, channel);
             } else {
-                logger.warn("received plaintext traffic on a encrypted channel, closing connection {}", ctx.getChannel());
+                logger.warn("received plaintext traffic on a encrypted channel, closing connection {}", channel);
             }
-            ctx.getChannel().close();
+            disconnectFromNodeChannel(channel, e);
-            disconnectFromNodeChannel(ctx.getChannel(), e.getCause());
+        } else if (isCloseDuringHandshakeException(e)) {
-        } else if (isCloseDuringHandshakeException(t)) {
             if (logger.isTraceEnabled()) {
-                logger.trace("connection {} closed during handshake", t, ctx.getChannel());
+                logger.trace("connection {} closed during handshake", e, channel);
             } else {
-                logger.warn("connection {} closed during handshake", ctx.getChannel());
+                logger.warn("connection {} closed during handshake", channel);
             }
-            ctx.getChannel().close();
+            disconnectFromNodeChannel(channel, e);
-            disconnectFromNodeChannel(ctx.getChannel(), e.getCause());
         } else {
-            super.exceptionCaught(ctx, e);
+            super.onException(channel, e);
         }
     }
 

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/user/User.java

@@ -35,8 +35,8 @@ public class User implements ToXContent {
     private final User runAs;
     private final Map<String, Object> metadata;
 
-    private final @Nullable String fullName;
+    @Nullable private final String fullName;
-    private final @Nullable String email;
+    @Nullable private final String email;
 
     public User(String username, String... roles) {
         this(username, roles, null, null, null);
@@ -294,6 +294,7 @@ public class User implements ToXContent {
     public interface Fields {
         ParseField USERNAME = new ParseField("username");
         ParseField PASSWORD = new ParseField("password");
+        ParseField PASSWORD_HASH = new ParseField("password_hash");
         ParseField ROLES = new ParseField("roles");
         ParseField FULL_NAME = new ParseField("full_name");
         ParseField EMAIL = new ParseField("email");

elasticsearch/x-pack/security/src/main/resources/org/elasticsearch/xpack/security/crypto/migrate.help

@@ -0,0 +1,34 @@
+NAME
+
+    migrate - Migrates elasticsearch file based users and roles to native realm
+
+SYNOPSIS
+
+    migrate native -U <url> [OPTIONS]
+
+DESCRIPTION
+
+    This tool migrates file based users[1] and roles[2] to the native realm in
+    elasticsearch, saving the administrator from needing to manually transition
+    them from the file.
+
+OPTIONS
+
+    -U --url         URL to connect to to user/role import
+
+    -u --username    [Optional] Username for authenticating with Elasticsearch
+
+    -p --password    [Optional] Password for authenticating with Elasticsearch
+
+    -n --users       [Optional] Users to migrate from file to native realm,
+                     if not specified all users will be migrated
+
+    -r --roles       [Optional] Roles to migrate from file to native realm
+                     if not specified all roles will be migrated
+
+    -c --config      [Optional] Path to Elasticsearch config directory
+
+NOTES
+
+    [1] https://www.elastic.co/guide/en/shield/current/setting-up-authentication.html
+    [2] https://www.elastic.co/guide/en/shield/current/configuring-rbac.html

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/http/netty/NettyHttpMockUtil.java

@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.http.netty;
 
-import org.elasticsearch.common.netty.OpenChannelsHandler;
+import org.elasticsearch.transport.netty.OpenChannelsHandler;
 
 import static org.mockito.Mockito.mock;
 

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java

@@ -8,6 +8,8 @@ package org.elasticsearch.integration;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.xpack.security.Security;
 import org.elasticsearch.xpack.security.authc.support.Hasher;
@@ -38,7 +40,8 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
                 "user1:" + USERS_PASSWD_HASHED + "\n" +
                 "user2:" + USERS_PASSWD_HASHED + "\n" +
                 "user3:" + USERS_PASSWD_HASHED + "\n" +
-                "user4:" + USERS_PASSWD_HASHED + "\n";
+                "user4:" + USERS_PASSWD_HASHED + "\n" +
+                "user5:" + USERS_PASSWD_HASHED + "\n";
     }
 
     @Override
@@ -47,7 +50,7 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
                 "role1:user1\n" +
                 "role2:user1,user4\n" +
                 "role3:user2,user4\n" +
-                "role4:user3,user4\n";
+                "role4:user3,user4,user5\n";
     }
 
     @Override
@@ -128,6 +131,33 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
         assertThat(response.getHits().getAt(1).getSource().get("field2").toString(), equalTo("value2"));
     }
 
+    public void testDLSIsAppliedBeforeFLS() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("test")
+                .addMapping("type1", "field1", "type=text", "field2", "type=text")
+        );
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1", "field2", "value1")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+        client().prepareIndex("test", "type1", "2").setSource("field1", "value2", "field2", "value2")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+
+        SearchResponse response = client().filterWithHeader(
+                Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user5", USERS_PASSWD)))
+                .prepareSearch("test").setQuery(QueryBuilders.termQuery("field1", "value2"))
+                .get();
+        assertHitCount(response, 1);
+        assertSearchHits(response, "2");
+        assertThat(response.getHits().getAt(0).getSource().size(), equalTo(1));
+        assertThat(response.getHits().getAt(0).getSource().get("field1").toString(), equalTo("value2"));
+
+        response = client().filterWithHeader(
+                Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user5", USERS_PASSWD)))
+                .prepareSearch("test").setQuery(QueryBuilders.termQuery("field1", "value1"))
+                .get();
+        assertHitCount(response, 0);
+    }
+
     public void testQueryCache() throws Exception {
         assertAcked(client().admin().indices().prepareCreate("test")
                         .setSettings(Settings.builder().put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true))

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/LicensingTests.java

@@ -144,41 +144,10 @@ public class LicensingTests extends SecurityIntegTestCase {
 
         disableLicensing();
 
-        try {
+        assertElasticsearchSecurityException(() -> client.admin().indices().prepareStats().get());
-            client.admin().indices().prepareStats().get();
+        assertElasticsearchSecurityException(() -> client.admin().cluster().prepareClusterStats().get());
-            fail("expected an license expired exception when executing an index stats action");
+        assertElasticsearchSecurityException(() -> client.admin().cluster().prepareHealth().get());
-        } catch (ElasticsearchSecurityException ee) {
+        assertElasticsearchSecurityException(() -> client.admin().cluster().prepareNodesStats().get());
-            // expected
-            assertThat(ee.getHeader("es.license.expired.feature"), hasItem(Security.NAME));
-            assertThat(ee.status(), is(RestStatus.UNAUTHORIZED));
-        }
-
-        try {
-            client.admin().cluster().prepareClusterStats().get();
-            fail("expected an license expired exception when executing cluster stats action");
-        } catch (ElasticsearchSecurityException ee) {
-            // expected
-            assertThat(ee.getHeader("es.license.expired.feature"), hasItem(Security.NAME));
-            assertThat(ee.status(), is(RestStatus.UNAUTHORIZED));
-        }
-
-        try {
-            client.admin().cluster().prepareHealth().get();
-            fail("expected an license expired exception when executing cluster health action");
-        } catch (ElasticsearchSecurityException ee) {
-            // expected
-            assertThat(ee.getHeader("es.license.expired.feature"), hasItem(Security.NAME));
-            assertThat(ee.status(), is(RestStatus.UNAUTHORIZED));
-        }
-
-        try {
-            client.admin().cluster().prepareNodesStats().get();
-            fail("expected an license expired exception when executing cluster health action");
-        } catch (ElasticsearchSecurityException ee) {
-            // expected
-            assertThat(ee.getHeader("es.license.expired.feature"), hasItem(Security.NAME));
-            assertThat(ee.status(), is(RestStatus.UNAUTHORIZED));
-        }
 
         enableLicensing(randomFrom(OperationMode.values()));
 
@@ -241,6 +210,12 @@ public class LicensingTests extends SecurityIntegTestCase {
         }
     }
 
+    private static void assertElasticsearchSecurityException(ThrowingRunnable runnable) {
+        ElasticsearchSecurityException ee = expectThrows(ElasticsearchSecurityException.class, runnable);
+        assertThat(ee.getHeader("es.license.expired.feature"), hasItem(Security.NAME));
+        assertThat(ee.status(), is(RestStatus.FORBIDDEN));
+    }
+
     public static void disableLicensing() {
         disableLicensing(OperationMode.BASIC);
     }

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java

@@ -37,7 +37,7 @@ import static org.hamcrest.Matchers.is;
  * This test assumes all subclass tests will be of type SUITE.  It picks a random realm configuration for the tests, and
  * writes a group to role mapping file for each node.
  */
-abstract public class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase {
+public abstract  class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase {
 
     public static final String XPACK_SECURITY_AUTHC_REALMS_EXTERNAL = "xpack.security.authc.realms.external";
     public static final String PASSWORD = "NickFuryHeartsES";
@@ -55,9 +55,9 @@ abstract public class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase
                     "Gods: [ \"cn=Gods,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com\" ] \n" +
                     "Philanthropists: [ \"cn=Philanthropists,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com\" ] \n";
 
-    static protected RealmConfig realmConfig;
+    protected static RealmConfig realmConfig;
-    static protected boolean useGlobalSSL;
+    protected static boolean useGlobalSSL;
-    static protected boolean sslEnabled;
+    protected static boolean sslEnabled;
 
     @BeforeClass
     public static void setupRealm() {