Merge branch 'master' into deprecate

Original commit: elastic/x-pack-elasticsearch@25985e9144

elasticsearch/qa/reindex-tests-with-security/build.gradle

@@ -2,6 +2,8 @@ apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
   testCompile project(path: ':x-plugins:elasticsearch:x-pack', configuration: 'runtime')
+  testCompile project(path: ':x-plugins:elasticsearch:x-pack', configuration: 'testArtifacts')
+  testCompile project(path: ':modules:reindex')
 }
 
 integTest {

elasticsearch/qa/reindex-tests-with-security/src/test/java/org/elasticsearch/xpack/security/ReindexWithSecurityIT.java

@@ -0,0 +1,128 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.reindex.BulkIndexByScrollResponse;
+import org.elasticsearch.index.reindex.DeleteByQueryAction;
+import org.elasticsearch.index.reindex.ReindexAction;
+import org.elasticsearch.index.reindex.ReindexPlugin;
+import org.elasticsearch.index.reindex.UpdateByQueryAction;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.SecurityIntegTestCase;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+
+public class ReindexWithSecurityIT extends SecurityIntegTestCase {
+
+    private boolean useSecurity3;
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        useSecurity3 = randomBoolean();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        Collection<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(ReindexPlugin.class);
+        return Collections.unmodifiableCollection(plugins);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        Collection<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(ReindexPlugin.class);
+        return Collections.unmodifiableCollection(plugins);
+    }
+
+    @Override
+    protected Settings externalClusterClientSettings() {
+        Settings.Builder builder = Settings.builder().put(super.externalClusterClientSettings());
+        if (useSecurity3) {
+            builder.put(NetworkModule.TRANSPORT_TYPE_KEY, Security.NAME3);
+        } else {
+            builder.put(NetworkModule.TRANSPORT_TYPE_KEY, Security.NAME4);
+        }
+        builder.put(Security.USER_SETTING.getKey(), "test_admin:changeme");
+        return builder.build();
+    }
+
+    public void testDeleteByQuery() {
+        createIndices("test1", "test2", "test3");
+
+        BulkIndexByScrollResponse response = DeleteByQueryAction.INSTANCE.newRequestBuilder(client()).source("test1", "test2").get();
+        assertNotNull(response);
+
+        response = DeleteByQueryAction.INSTANCE.newRequestBuilder(client()).source("test*").get();
+        assertNotNull(response);
+
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class,
+                () -> DeleteByQueryAction.INSTANCE.newRequestBuilder(client()).source("test1", "index1").get());
+        assertEquals("no such index", e.getMessage());
+    }
+
+    public void testUpdateByQuery() {
+        createIndices("test1", "test2", "test3");
+
+        BulkIndexByScrollResponse response = UpdateByQueryAction.INSTANCE.newRequestBuilder(client()).source("test1", "test2").get();
+        assertNotNull(response);
+
+        response = UpdateByQueryAction.INSTANCE.newRequestBuilder(client()).source("test*").get();
+        assertNotNull(response);
+
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class,
+                () -> UpdateByQueryAction.INSTANCE.newRequestBuilder(client()).source("test1", "index1").get());
+        assertEquals("no such index", e.getMessage());
+    }
+
+    public void testReindex() {
+        createIndices("test1", "test2", "test3", "dest");
+
+        BulkIndexByScrollResponse response = ReindexAction.INSTANCE.newRequestBuilder(client()).source("test1", "test2")
+                .destination("dest").get();
+        assertNotNull(response);
+
+        response = ReindexAction.INSTANCE.newRequestBuilder(client()).source("test*").destination("dest").get();
+        assertNotNull(response);
+
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class,
+                () -> ReindexAction.INSTANCE.newRequestBuilder(client()).source("test1", "index1").destination("dest").get());
+        assertEquals("no such index", e.getMessage());
+    }
+
+    private void createIndices(String... indices) {
+        if (randomBoolean()) {
+            //no aliases
+            createIndex(indices);
+        } else {
+            if (randomBoolean()) {
+                //one alias per index with suffix "-alias"
+                for (String index : indices) {
+                    client().admin().indices().prepareCreate(index).setSettings(indexSettings()).addAlias(new Alias(index + "-alias"));
+                }
+            } else {
+                //same alias pointing to all indices
+                for (String index : indices) {
+                    client().admin().indices().prepareCreate(index).setSettings(indexSettings()).addAlias(new Alias("alias"));
+                }
+            }
+        }
+
+        for (String index : indices) {
+            client().prepareIndex(index, "type").setSource("field", "value").get();
+        }
+        refresh();
+    }
+}

elasticsearch/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java

@@ -13,11 +13,8 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
 import org.elasticsearch.xpack.security.action.role.GetRolesResponse;
 import org.elasticsearch.xpack.security.action.user.GetUsersResponse;
 import org.elasticsearch.xpack.security.action.user.PutUserResponse;
@@ -123,7 +120,7 @@ public class MigrateToolIT extends MigrateToolTestCase {
                 .timeout(TimeValue.timeValueSeconds(30))
                 .waitForYellowStatus()
                 .waitForEvents(Priority.LANGUID)
-                .waitForRelocatingShards(0))
+                .waitForNoRelocatingShards(true))
                 .actionGet();
         SearchResponse searchResp = client.filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("index1").get();
     }

elasticsearch/qa/smoke-test-plugins-ssl/build.gradle

@@ -1,5 +1,14 @@
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.MavenFilteringHack
+import org.elasticsearch.gradle.test.NodeInfo
+
+import javax.net.ssl.HttpsURLConnection
+import javax.net.ssl.KeyManagerFactory
+import javax.net.ssl.SSLContext
+import javax.net.ssl.TrustManagerFactory
+import java.nio.charset.StandardCharsets
+import java.security.KeyStore
+import java.security.SecureRandom
 
 apply plugin: 'elasticsearch.rest-test'
 
@@ -177,25 +186,47 @@ integTest {
     setupCommand 'setupMonitoringUser',
                  'bin/x-pack/users', 'useradd', 'monitoring_agent', '-p', 'changeme', '-r', 'remote_monitoring_agent'
 
-    // Required to detect that the monitoring agent service has started
+    waitCondition = { NodeInfo node, AntBuilder ant ->
-    setting 'logger.level', 'DEBUG'
+      File tmpFile = new File(node.cwd, 'wait.success')
-
+      KeyStore keyStore = KeyStore.getInstance("JKS");
-    waitCondition = { node, ant ->
+      keyStore.load(clientKeyStore.newInputStream(), 'keypass'.toCharArray());
-      // HTTPS check is tricky to do, so we wait for the log file to indicate that the node is started
+      KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
-      String waitForNodeStartProp = "waitForNodeStart${name}"
+      kmf.init(keyStore, 'keypass'.toCharArray());
-      ant.waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '100', checkeveryunit: 'millisecond',
+      TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
-              timeoutproperty: waitForNodeStartProp) {
+      tmf.init(keyStore);
-        and {
+      SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
-          resourcecontains(resource: "${node.startLog.toString()}", substring: 'started')
+      sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), new SecureRandom());
-          resourcecontains(resource: "${node.startLog.toString()}", substring: 'monitoring service started')
+      for (int i = 0; i < 10; i++) {
+        // we use custom wait logic here for HTTPS
+        HttpsURLConnection httpURLConnection = null;
+        try {
+          httpURLConnection = (HttpsURLConnection) new URL("https://${node.httpUri()}/_cluster/health?wait_for_nodes=${numNodes}").openConnection();
+          httpURLConnection.setSSLSocketFactory(sslContext.getSocketFactory());
+          httpURLConnection.setRequestProperty("Authorization", "Basic " +
+                  Base64.getEncoder().encodeToString("test_user:changeme".getBytes(StandardCharsets.UTF_8)));
+          httpURLConnection.setRequestMethod("GET");
+          httpURLConnection.connect();
+          if (httpURLConnection.getResponseCode() == 200) {
+            tmpFile.withWriter StandardCharsets.UTF_8.name(), {
+              it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name()))
+            }
+          }
+        } catch (IOException e) {
+          if (i == 9) {
+            logger.error("final attempt of calling cluster health failed", e)
+          } else {
+            logger.debug("failed to call cluster health", e)
+          }
+        } finally {
+          if (httpURLConnection != null) {
+            httpURLConnection.disconnect();
           }
         }
 
-      if (ant.project.getProperty(waitForNodeStartProp)) {
+        // did not start, so wait a bit before trying again
-        println "Timed out when looking for node startup in log file ${node.startLog.toString()}"
+        Thread.sleep(500L);
-        return false;
       }
-      return true;
+      return tmpFile.exists()
     }
   }
 }

elasticsearch/x-pack/build.gradle

@@ -176,7 +176,7 @@ integTest {
           // we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned
           HttpURLConnection httpURLConnection = null;
           try {
-            httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}").openConnection();
+            httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=${numNodes}").openConnection();
             httpURLConnection.setRequestProperty("Authorization", "Basic " +
                     Base64.getEncoder().encodeToString("elastic:changeme".getBytes(StandardCharsets.UTF_8)));
             httpURLConnection.setRequestMethod("GET");
@@ -187,7 +187,11 @@ integTest {
               }
             }
           } catch (Exception e) {
-            e.printStackTrace()
+            if (i == 9) {
+              logger.error("final attempt of calling cluster health failed", e)
+            } else {
+              logger.debug("failed to call cluster health", e)
+            }
           } finally {
             if (httpURLConnection != null) {
               httpURLConnection.disconnect();

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/TribeTransportTestCase.java

@@ -203,7 +203,7 @@ public abstract class TribeTransportTestCase extends ESIntegTestCase {
     private void ensureYellow(TestCluster testCluster) {
         ClusterHealthResponse actionGet = testCluster.client().admin().cluster()
                         .health(Requests.clusterHealthRequest().waitForYellowStatus()
-                                .waitForEvents(Priority.LANGUID).waitForRelocatingShards(0)).actionGet();
+                                .waitForEvents(Priority.LANGUID).waitForNoRelocatingShards(true)).actionGet();
         if (actionGet.isTimedOut()) {
             logger.info("ensureGreen timed out, cluster state:\n{}\n{}", testCluster.client().admin().cluster()
                     .prepareState().get().getState().prettyPrint(),

elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/agent/AgentService.java

@@ -119,7 +119,6 @@ public class AgentService extends AbstractLifecycleComponent {
 
     @Override
     protected void doStart() {
-        // Please don't remove this log message since it can be used in integration tests
         logger.debug("monitoring service started");
 
         for (Collector collector : collectors) {

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -302,7 +302,7 @@ public class Security implements ActionPlugin, IngestPlugin {
         components.add(authcService);
 
         final FileRolesStore fileRolesStore = new FileRolesStore(settings, env, resourceWatcherService);
-        final NativeRolesStore nativeRolesStore = new NativeRolesStore(settings, client, threadPool);
+        final NativeRolesStore nativeRolesStore = new NativeRolesStore(settings, client);
         final ReservedRolesStore reservedRolesStore = new ReservedRolesStore(securityContext);
         final CompositeRolesStore allRolesStore = new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore, reservedRolesStore);
         final AuthorizationService authzService = new AuthorizationService(settings, allRolesStore, clusterService,

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/indicesresolver/DefaultIndicesAndAliasesResolver.java

@@ -49,13 +49,10 @@ public class DefaultIndicesAndAliasesResolver implements IndicesAndAliasesResolv
 
     @Override
     public Set<String> resolve(User user, String action, TransportRequest request, MetaData metaData) {
-
         boolean isIndicesRequest = request instanceof CompositeIndicesRequest || request instanceof IndicesRequest;
-        assert isIndicesRequest : "Request [" + request + "] is not an Indices request, but should be.";
-
         // if for some reason we are missing an action... just for safety we'll reject
-        if (!isIndicesRequest) {
+        if (isIndicesRequest == false) {
-            return Collections.emptySet();
+            throw new IllegalStateException("Request [" + request + "] is not an Indices request, but should be.");
         }
 
         if (request instanceof CompositeIndicesRequest) {
@@ -74,7 +71,7 @@ public class DefaultIndicesAndAliasesResolver implements IndicesAndAliasesResolv
         final Set<String> indices;
         if (indicesRequest instanceof PutMappingRequest
                 && ((PutMappingRequest) indicesRequest).getConcreteIndex() != null) {
-            /**
+            /*
              * This is a special case since PutMappingRequests from dynamic mapping updates have a concrete index
              * if this index is set and it's in the list of authorized indices we are good and don't need to put
              * the list of indices in there, if we do so it will result in an invalid request and the update will fail.

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java

@@ -9,16 +9,14 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.BiFunction;
+import java.util.concurrent.locks.ReadWriteLock;
-import java.util.function.Function;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
@@ -38,18 +36,19 @@ import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.cache.Cache;
+import org.elasticsearch.common.cache.CacheBuilder;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.ReleasableLock;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -57,8 +56,6 @@ import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.threadpool.ThreadPool.Names;
 import org.elasticsearch.xpack.security.InternalClient;
 import org.elasticsearch.xpack.security.SecurityTemplateService;
 import org.elasticsearch.xpack.security.action.role.ClearRolesCacheRequest;
@@ -69,7 +66,6 @@ import org.elasticsearch.xpack.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.security.authz.permission.IndicesPermission.Group;
 import org.elasticsearch.xpack.security.authz.permission.Role;
 import org.elasticsearch.xpack.security.client.SecurityClient;
-import org.elasticsearch.threadpool.ThreadPool.Cancellable;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.security.Security.setting;
@@ -91,8 +87,10 @@ public class NativeRolesStore extends AbstractComponent implements RolesStore, C
     public static final Setting<TimeValue> SCROLL_KEEP_ALIVE_SETTING =
             Setting.timeSetting(setting("authz.store.roles.index.scroll.keep_alive"), TimeValue.timeValueSeconds(10L), Property.NodeScope);
 
-    public static final Setting<TimeValue> POLL_INTERVAL_SETTING =
+    private static final Setting<Integer> CACHE_SIZE_SETTING =
-            Setting.timeSetting(setting("authz.store.roles.index.reload.interval"), TimeValue.timeValueSeconds(30L), Property.NodeScope);
+            Setting.intSetting(setting("authz.store.roles.index.cache.max_size"), 10000, Property.NodeScope);
+    private static final Setting<TimeValue> CACHE_TTL_SETTING =
+            Setting.timeSetting(setting("authz.store.roles.index.cache.ttl"), TimeValue.timeValueMinutes(20), Property.NodeScope);
 
     public enum State {
         INITIALIZED,
@@ -106,21 +104,34 @@ public class NativeRolesStore extends AbstractComponent implements RolesStore, C
     public static final String ROLE_DOC_TYPE = "role";
 
     private final InternalClient client;
-    private final ThreadPool threadPool;
     private final AtomicReference<State> state = new AtomicReference<>(State.INITIALIZED);
-    private final ConcurrentHashMap<String, RoleAndVersion> roleCache = new ConcurrentHashMap<>();
+    private final Cache<String, RoleAndVersion> roleCache;
+    // the lock is used in an odd manner; when iterating over the cache we cannot have modifiers other than deletes using
+    // the iterator but when not iterating we can modify the cache without external locking. When making normal modifications to the cache
+    // the read lock is obtained so that we can allow concurrent modifications; however when we need to iterate over the keys or values of
+    // the cache the write lock must obtained to prevent any modifications
+    private final ReleasableLock readLock;
+    private final ReleasableLock writeLock;
+
+    {
+        final ReadWriteLock iterationLock = new ReentrantReadWriteLock();
+        readLock = new ReleasableLock(iterationLock.readLock());
+        writeLock = new ReleasableLock(iterationLock.writeLock());
+    }
 
     private SecurityClient securityClient;
     private int scrollSize;
     private TimeValue scrollKeepAlive;
-    private Cancellable pollerCancellable;
 
     private volatile boolean securityIndexExists = false;
 
-    public NativeRolesStore(Settings settings, InternalClient client, ThreadPool threadPool) {
+    public NativeRolesStore(Settings settings, InternalClient client) {
         super(settings);
         this.client = client;
-        this.threadPool = threadPool;
+        this.roleCache = CacheBuilder.<String, RoleAndVersion>builder()
+                .setMaximumWeight(CACHE_SIZE_SETTING.get(settings))
+                .setExpireAfterWrite(CACHE_TTL_SETTING.get(settings).getMillis())
+                .build();
     }
 
     public boolean canStart(ClusterState clusterState, boolean master) {
@@ -144,15 +155,6 @@ public class NativeRolesStore extends AbstractComponent implements RolesStore, C
                 this.securityClient = new SecurityClient(client);
                 this.scrollSize = SCROLL_SIZE_SETTING.get(settings);
                 this.scrollKeepAlive = SCROLL_KEEP_ALIVE_SETTING.get(settings);
-                TimeValue pollInterval = POLL_INTERVAL_SETTING.get(settings);
-                RolesStorePoller poller = new RolesStorePoller();
-                try {
-                    poller.doRun();
-                } catch (Exception e) {
-                    logger.warn("failed to perform initial poll of roles index [{}]. scheduling again in [{}]", e,
-                            SecurityTemplateService.SECURITY_INDEX_NAME, pollInterval);
-                }
-                pollerCancellable = threadPool.scheduleWithFixedDelay(poller, pollInterval, Names.GENERIC);
                 state.set(State.STARTED);
             }
         } catch (Exception e) {
@@ -163,13 +165,9 @@ public class NativeRolesStore extends AbstractComponent implements RolesStore, C
 
     public void stop() {
         if (state.compareAndSet(State.STARTED, State.STOPPING)) {
-            try {
-                pollerCancellable.cancel();
-            } finally {
             state.set(State.STOPPED);
         }
     }
-    }
 
     /**
      * Retrieve a list of roles, if rolesToGet is null or empty, fetch all roles
@@ -341,7 +339,8 @@ public class NativeRolesStore extends AbstractComponent implements RolesStore, C
             return usageStats;
         }
 
-        long count = (long) roleCache.size();
+        long count = roleCache.count();
+        try (final ReleasableLock ignored = writeLock.acquire()) {
             for (RoleAndVersion rv : roleCache.values()) {
                 Role role = rv.getRole();
                 for (Group group : role.indices()) {
@@ -352,6 +351,7 @@ public class NativeRolesStore extends AbstractComponent implements RolesStore, C
                     break;
                 }
             }
+        }
 
         // slow path - query for necessary information
         if (fls == false || dls == false) {
@@ -408,9 +408,7 @@ public class NativeRolesStore extends AbstractComponent implements RolesStore, C
         final AtomicReference<GetResponse> getRef = new AtomicReference<>(null);
         final CountDownLatch latch = new CountDownLatch(1);
         try {
-            roleAndVersion = roleCache.computeIfAbsent(roleId, new Function<String, RoleAndVersion>() {
+            roleAndVersion = roleCache.computeIfAbsent(roleId, (key) -> {
-                @Override
-                public RoleAndVersion apply(String key) {
                 logger.debug("attempting to load role [{}] from index", key);
                 executeGetRoleRequest(roleId, new LatchedActionListener<>(new ActionListener<GetResponse>() {
                     @Override
@@ -444,11 +442,16 @@ public class NativeRolesStore extends AbstractComponent implements RolesStore, C
                     return null;
                 }
                 logger.debug("loaded role [{}] from index with version [{}]", key, response.getVersion());
+                try (final ReleasableLock ignored = readLock.acquire()) {
                     return new RoleAndVersion(descriptor, response.getVersion());
                 }
             });
-        } catch (RuntimeException e) {
+        } catch (ExecutionException e) {
-            logger.error("could not get or load value from cache for role [{}]", e, roleId);
+            if (e.getCause() instanceof NullPointerException) {
+                logger.trace("role [{}] was not found", e, roleId);
+            } else {
+                logger.error("failed to load role [{}]", e, roleId);
+            }
         }
 
         return roleAndVersion;
@@ -490,19 +493,23 @@ public class NativeRolesStore extends AbstractComponent implements RolesStore, C
         if (state != State.STOPPED && state != State.FAILED) {
             throw new IllegalStateException("can only reset if stopped!!!");
         }
-        this.roleCache.clear();
+        invalidateAll();
         this.securityIndexExists = false;
         this.state.set(State.INITIALIZED);
     }
 
     public void invalidateAll() {
         logger.debug("invalidating all roles in cache");
-        roleCache.clear();
+        try (final ReleasableLock ignored = readLock.acquire()) {
+            roleCache.invalidateAll();
+        }
     }
 
     public void invalidate(String role) {
         logger.debug("invalidating role [{}] in cache", role);
-        roleCache.remove(role);
+        try (final ReleasableLock ignored = readLock.acquire()) {
+            roleCache.invalidate(role);
+        }
     }
 
     private <Response> void clearRoleCache(final String role, ActionListener<Response> listener, Response response) {
@@ -560,97 +567,6 @@ public class NativeRolesStore extends AbstractComponent implements RolesStore, C
         }
     }
 
-    private class RolesStorePoller extends AbstractRunnable {
-
-        @Override
-        protected void doRun() throws Exception {
-            if (isStopped()) {
-                return;
-            }
-            if (securityIndexExists == false) {
-                logger.trace("cannot poll for role changes since security index [{}] does not exist",
-                        SecurityTemplateService.SECURITY_INDEX_NAME);
-                return;
-            }
-
-            // hold a reference to the client since the poller may run after the class is stopped (we don't interrupt it running) and
-            // we reset when we test which sets the client to null...
-            final Client client = NativeRolesStore.this.client;
-
-            logger.trace("starting polling of roles index to check for changes");
-            SearchResponse response = null;
-            // create a copy of the keys in the cache since we will be modifying this list
-            final Set<String> existingRoles = new HashSet<>(roleCache.keySet());
-            try {
-                client.admin().indices().prepareRefresh(SecurityTemplateService.SECURITY_INDEX_NAME);
-                SearchRequest request = client.prepareSearch(SecurityTemplateService.SECURITY_INDEX_NAME)
-                        .setScroll(scrollKeepAlive)
-                        .setQuery(QueryBuilders.typeQuery(ROLE_DOC_TYPE))
-                        .setSize(scrollSize)
-                        .setFetchSource(true)
-                        .setVersion(true)
-                        .request();
-                response = client.search(request).actionGet();
-
-                boolean keepScrolling = response.getHits().getHits().length > 0;
-                while (keepScrolling) {
-                    if (isStopped()) {
-                        return;
-                    }
-                    for (SearchHit hit : response.getHits().getHits()) {
-                        final String roleName = hit.getId();
-                        final long version = hit.version();
-                        existingRoles.remove(roleName);
-                        // we use the locking mechanisms provided by the map/cache to help protect against concurrent operations
-                        // that will leave the cache in a bad state
-                        roleCache.computeIfPresent(roleName, new BiFunction<String, RoleAndVersion, RoleAndVersion>() {
-                            @Override
-                            public RoleAndVersion apply(String roleName, RoleAndVersion existing) {
-                                if (version > existing.getVersion()) {
-                                    RoleDescriptor rd = transformRole(hit.getId(), hit.getSourceRef());
-                                    if (rd != null) {
-                                        return new RoleAndVersion(rd, version);
-                                    }
-                                }
-                                return existing;
-                            }
-                        });
-                    }
-                    SearchScrollRequest scrollRequest = client.prepareSearchScroll(response.getScrollId())
-                            .setScroll(scrollKeepAlive).request();
-                    response = client.searchScroll(scrollRequest).actionGet();
-                    keepScrolling = response.getHits().getHits().length > 0;
-                }
-
-                // check to see if we had roles that do not exist in the index
-                if (existingRoles.isEmpty() == false) {
-                    for (String roleName : existingRoles) {
-                        logger.trace("role [{}] does not exist anymore, removing from cache", roleName);
-                        roleCache.remove(roleName);
-                    }
-                }
-            } catch (IndexNotFoundException e) {
-                logger.trace("security index does not exist", e);
-            } finally {
-                if (response != null) {
-                    ClearScrollRequest clearScrollRequest = client.prepareClearScroll().addScrollId(response.getScrollId()).request();
-                    client.clearScroll(clearScrollRequest).actionGet();
-                }
-            }
-            logger.trace("completed polling of roles index");
-        }
-
-        @Override
-        public void onFailure(Exception e) {
-            logger.error("error occurred while checking the native roles for changes", e);
-        }
-
-        private boolean isStopped() {
-            State state = state();
-            return state == State.STOPPED || state == State.STOPPING;
-        }
-    }
-
     private static class RoleAndVersion {
 
         private final RoleDescriptor roleDescriptor;
@@ -679,6 +595,7 @@ public class NativeRolesStore extends AbstractComponent implements RolesStore, C
     public static void addSettings(List<Setting<?>> settings) {
         settings.add(SCROLL_SIZE_SETTING);
         settings.add(SCROLL_KEEP_ALIVE_SETTING);
-        settings.add(POLL_INTERVAL_SETTING);
+        settings.add(CACHE_SIZE_SETTING);
+        settings.add(CACHE_TTL_SETTING);
     }
 }

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4Transport.java

@@ -11,10 +11,7 @@ import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelOutboundHandlerAdapter;
 import io.netty.channel.ChannelPromise;
 import io.netty.handler.ssl.SslHandler;
-import io.netty.util.concurrent.Future;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.inject.internal.Nullable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -35,9 +32,6 @@ import javax.net.ssl.SSLParameters;
 
 import java.net.InetSocketAddress;
 import java.net.SocketAddress;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.xpack.security.Security.setting;
 import static org.elasticsearch.xpack.security.Security.settingPrefix;
@@ -131,28 +125,6 @@ public class SecurityNetty4Transport extends Netty4Transport {
         return new SecurityClientChannelInitializer();
     }
 
-    /**
-     * This method ensures that all channels have their SSL handshakes completed. This is necessary to prevent the application from
-     * writing data while the handshake is in progress which could cause the handshake to fail.
-     */
-    @Override
-    protected void onAfterChannelsConnected(NodeChannels nodeChannels) {
-        List<Tuple<Future<Channel>, Channel>> handshakes = new ArrayList<>();
-        for (Channel channel : nodeChannels.allChannels) {
-            SslHandler handler = channel.pipeline().get(SslHandler.class);
-            if (handler != null) {
-                handshakes.add(Tuple.tuple(handler.handshakeFuture(), channel));
-            }
-        }
-
-        for (Tuple<Future<Channel>, Channel> handshake : handshakes) {
-            handshake.v1().awaitUninterruptibly(30L, TimeUnit.SECONDS);
-            if (!handshake.v1().isSuccess()) {
-                throw new ElasticsearchException("handshake failed for channel [{}]", handshake.v2());
-            }
-        }
-    }
-
     class SecurityServerChannelInitializer extends ServerChannelInitializer {
 
         private final boolean sslEnabled;

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java

@@ -5,25 +5,14 @@
  */
 package org.elasticsearch.integration;
 
-import org.apache.http.message.BasicHeader;
-import org.elasticsearch.action.DocWriteResponse;
-import org.elasticsearch.action.delete.DeleteResponse;
-import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.NativeRealmIntegTestCase;
-import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.action.role.DeleteRoleResponse;
 import org.elasticsearch.xpack.security.action.role.GetRolesResponse;
 import org.elasticsearch.xpack.security.action.role.PutRoleResponse;
-import org.elasticsearch.xpack.security.authc.support.SecuredString;
-import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
 import org.elasticsearch.xpack.security.client.SecurityClient;
 import org.junit.Before;
@@ -34,14 +23,12 @@ import java.util.List;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
-import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
 
 /**
- * Test for the clear roles API that changes the polling aspect of security to only run once an hour in order to
+ * Test for the clear roles API
- * test the cache clearing APIs.
  */
 public class ClearRolesCacheTests extends NativeRealmIntegTestCase {
 
@@ -79,11 +66,8 @@ public class ClearRolesCacheTests extends NativeRealmIntegTestCase {
 
     @Override
     public Settings nodeSettings(int nodeOrdinal) {
-        TimeValue pollerInterval = TimeValue.timeValueMillis((long) randomIntBetween(2, 2000));
-        logger.debug("using poller interval [{}]", pollerInterval);
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(NativeRolesStore.POLL_INTERVAL_SETTING.getKey(), pollerInterval.getStringRep())
                 .put(NetworkModule.HTTP_ENABLED.getKey(), true)
                 .build();
     }
@@ -109,67 +93,17 @@ public class ClearRolesCacheTests extends NativeRealmIntegTestCase {
         assertRolesAreCorrect(securityClient, toModify);
     }
 
-    public void testModifyingDocumentsDirectly() throws Exception {
+    public void testDeletingViaApiClearsCache() throws Exception {
-        int modifiedRolesCount = randomIntBetween(1, roles.length);
+        final int rolesToDelete = randomIntBetween(1, roles.length - 1);
-        List<String> toModify = randomSubsetOf(modifiedRolesCount, roles);
+        List<String> toDelete = randomSubsetOf(rolesToDelete, roles);
-        logger.debug("--> modifying roles {} to have run_as", toModify);
+        for (String role : toDelete) {
-        final boolean refresh = randomBoolean();
+            DeleteRoleResponse response = securityClient().prepareDeleteRole(role).get();
-        for (String role : toModify) {
+            assertTrue(response.found());
-            UpdateResponse response = internalClient().prepareUpdate().setId(role).setIndex(SecurityTemplateService.SECURITY_INDEX_NAME)
-                    .setType(NativeRolesStore.ROLE_DOC_TYPE)
-                    .setDoc("run_as", new String[] { role })
-                    .setRefreshPolicy(refresh ? IMMEDIATE : NONE)
-                    .get();
-            assertEquals(DocWriteResponse.Result.UPDATED, response.getResult());
-            logger.debug("--> updated role [{}] with run_as", role);
         }
 
-        // in this test, the poller runs too frequently to check the cache still has roles without run as
+        GetRolesResponse roleResponse = securityClient().prepareGetRoles().names(roles).get();
-        // clear the cache and we should definitely see the latest values!
+        assertTrue(roleResponse.hasRoles());
-        SecurityClient securityClient = securityClient(internalCluster().transportClient());
+        assertThat(roleResponse.roles().length, is(roles.length - rolesToDelete));
-        final boolean useHttp = randomBoolean();
-        final boolean clearAll = randomBoolean();
-        logger.debug("--> starting to clear roles. using http [{}] clearing all [{}]", useHttp, clearAll);
-        String[] rolesToClear = clearAll ? (randomBoolean() ? roles : null) : toModify.toArray(new String[toModify.size()]);
-        if (useHttp) {
-            String path;
-            if (rolesToClear == null) {
-                path = "/_xpack/security/role/" + (randomBoolean() ? "*" : "_all") + "/_clear_cache";
-            } else {
-                path = "/_xpack/security/role/" + Strings.arrayToCommaDelimitedString(rolesToClear) + "/_clear_cache";
-            }
-            Response response = getRestClient().performRequest("POST", path,
-                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                            UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME,
-                                    new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))));
-            assertThat(response.getStatusLine().getStatusCode(), is(RestStatus.OK.getStatus()));
-        } else {
-            securityClient.prepareClearRolesCache().names(rolesToClear).get();
-        }
-
-        assertRolesAreCorrect(securityClient, toModify);
-    }
-
-    public void testDeletingRoleDocumentDirectly() throws Exception {
-        SecurityClient securityClient = securityClient(internalCluster().transportClient());
-
-        final String role = randomFrom(roles);
-        RoleDescriptor[] foundRoles = securityClient.prepareGetRoles().names(role).get().roles();
-        assertThat(foundRoles.length, is(1));
-        logger.debug("--> deleting role [{}]", role);
-        final boolean refresh = randomBoolean();
-        DeleteResponse response = internalClient()
-                .prepareDelete(SecurityTemplateService.SECURITY_INDEX_NAME, NativeRolesStore.ROLE_DOC_TYPE, role)
-                .setRefreshPolicy(refresh ? IMMEDIATE : NONE)
-                .get();
-        assertEquals(DocWriteResponse.Result.DELETED, response.getResult());
-
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat(securityClient.prepareGetRoles().names(role).get().roles(), arrayWithSize(0));
-            }
-        });
     }
 
     private void assertRolesAreCorrect(SecurityClient securityClient, List<String> toModify) {

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java

@@ -153,7 +153,7 @@ public abstract class SecurityIntegTestCase extends ESIntegTestCase {
             // TODO: disable this assertion for now, due to random runs with mock plugins. perhaps run without mock plugins?
 //            assertThat(nodeInfo.getPlugins().getInfos(), hasSize(2));
             Collection<String> pluginNames =
-                    nodeInfo.getPlugins().getPluginInfos().stream().map(p -> p.getName()).collect(Collectors.toList());
+                    nodeInfo.getPlugins().getPluginInfos().stream().map(p -> p.getClassname()).collect(Collectors.toList());
             assertThat("plugin [" + xpackPluginClass().getName() + "] not found in [" + pluginNames + "]", pluginNames,
                 hasItem(xpackPluginClass().getName()));
         }

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java

@@ -736,7 +736,7 @@ public class IndexAuditTrailTests extends SecurityIntegTestCase {
 
         // pretty ugly but just a rip of ensureYellow that uses a different client
         ClusterHealthResponse actionGet = getClient().admin().cluster().health(Requests.clusterHealthRequest(indices)
-                .waitForRelocatingShards(0).waitForYellowStatus().waitForEvents(Priority.LANGUID)).actionGet();
+                .waitForNoRelocatingShards(true).waitForYellowStatus().waitForEvents(Priority.LANGUID)).actionGet();
         if (actionGet.isTimedOut()) {
             logger.info("ensureYellow timed out, cluster state:\n{}\n{}",
                     getClient().admin().cluster().prepareState().get().getState().prettyPrint(),

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authz/indicesresolver/IndicesAndAliasesResolverIntegrationTests.java

@@ -24,9 +24,9 @@ import static org.elasticsearch.test.SecurityTestsUtils.assertAuthorizationExcep
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItems;
-import static org.hamcrest.Matchers.is;
 
 public class IndicesAndAliasesResolverIntegrationTests extends SecurityIntegTestCase {
+
     @Override
     protected String configRoles() {
         return SecuritySettingsSource.DEFAULT_ROLE + ":\n" +
@@ -57,50 +57,30 @@ public class IndicesAndAliasesResolverIntegrationTests extends SecurityIntegTest
     public void testSearchNonAuthorizedWildcard() {
         //wildcard doesn't match any authorized index
         createIndices("test1", "test2", "index1", "index2");
-        try {
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () -> client().prepareSearch("index*").get());
-            client().prepareSearch("index*").get();
+        assertEquals("no such index", e.getMessage());
-            fail("Expected IndexNotFoundException");
-        } catch (IndexNotFoundException e) {
-            assertThat(e.getMessage(), is("no such index"));
-        }
     }
 
     public void testEmptyClusterSearchForAll() {
-        try {
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () -> client().prepareSearch().get());
-            client().prepareSearch().get();
+        assertEquals("no such index", e.getMessage());
-            fail("Expected IndexNotFoundException");
-        } catch (IndexNotFoundException e) {
-            assertThat(e.getMessage(), is("no such index"));
-        }
     }
 
     public void testEmptyClusterSearchForWildcard() {
-        try {
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () -> client().prepareSearch("*").get());
-            client().prepareSearch("*").get();
+        assertEquals("no such index", e.getMessage());
-            fail("Expected IndexNotFoundException");
-        } catch (IndexNotFoundException e) {
-            assertThat(e.getMessage(), is("no such index"));
-        }
     }
 
     public void testEmptyAuthorizedIndicesSearchForAll() {
         createIndices("index1", "index2");
-        try {
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () -> client().prepareSearch().get());
-            client().prepareSearch().get();
+        assertEquals("no such index", e.getMessage());
-            fail("Expected IndexNotFoundException");
-        } catch (IndexNotFoundException e) {
-            assertThat(e.getMessage(), is("no such index"));
-        }
     }
 
     public void testEmptyAuthorizedIndicesSearchForWildcard() {
         createIndices("index1", "index2");
-        try {
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () -> client().prepareSearch("*").get());
-            client().prepareSearch("*").get();
+        assertEquals("no such index", e.getMessage());
-            fail("Expected IndexNotFoundException");
-        } catch (IndexNotFoundException e) {
-            assertThat(e.getMessage(), is("no such index"));
-        }
     }
 
     public void testExplicitNonAuthorizedIndex() {
@@ -187,14 +167,10 @@ public class IndicesAndAliasesResolverIntegrationTests extends SecurityIntegTest
     public void testMultiSearchWildcard() {
         //test4 is missing but authorized, only that specific item fails
         createIndices("test1", "test2", "test3", "index1");
-        try {
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class,
-            client().prepareMultiSearch()
+                () -> client().prepareMultiSearch().add(Requests.searchRequest())
-                    .add(Requests.searchRequest())
+                        .add(Requests.searchRequest("index*")).get());
-                    .add(Requests.searchRequest("index*")).get();
+        assertEquals("no such index", e.getMessage());
-            fail("Expected IndexNotFoundException");
-        } catch (IndexNotFoundException e) {
-            assertThat(e.getMessage(), is("no such index"));
-        }
     }
 
     private static void assertReturnedIndices(SearchResponse searchResponse, String... indices) {
@@ -207,13 +183,9 @@ public class IndicesAndAliasesResolverIntegrationTests extends SecurityIntegTest
     }
 
     private static void assertThrowsAuthorizationException(ActionRequestBuilder actionRequestBuilder) {
-        try {
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, actionRequestBuilder::get);
-            actionRequestBuilder.get();
-            fail("search should fail due to attempt to access non authorized indices");
-        } catch(ElasticsearchSecurityException e) {
         assertAuthorizationException(e, containsString("is unauthorized for user ["));
     }
-    }
 
     private void createIndices(String... indices) {
         if (randomBoolean()) {
@@ -233,7 +205,6 @@ public class IndicesAndAliasesResolverIntegrationTests extends SecurityIntegTest
             }
         }
 
-        ensureGreen();
         for (String index : indices) {
             client().prepareIndex(index, "type").setSource("field", "value").get();
         }