[Security] Cross cluster wildcard security (elastic/x-pack-elasticsearch#1290)

Support the resolution of remote index names, including those that contain wildcards in the cluster name or index part)

Specifically these work:
- `GET /remote*:foo/_search`
- `GET /*:foo/_search`
- `GET /*:foo,*/_search`
- `GET /remote:*/_search`
- `GET /*:*/_search`

This change assumes that every user is allowed to attempt a cross-cluster search against any remote index, and the actual authorisation of indices happens on the remote nodes. Thus ` GET /*:foo/_search` will expand to search the `foo` index on every registered remote without consideration of the roles and privileges that the user has on the source cluster.

Original commit: elastic/x-pack-elasticsearch@b45041aaa3

plugin/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java

@@ -102,7 +102,7 @@ public class AuthorizationService extends AbstractComponent {
         this.rolesStore = rolesStore;
         this.clusterService = clusterService;
         this.auditTrail = auditTrail;
-        this.indicesAndAliasesResolver = new IndicesAndAliasesResolver(new IndexNameExpressionResolver(settings));
+        this.indicesAndAliasesResolver = new IndicesAndAliasesResolver(settings, clusterService);
         this.authcFailureHandler = authcFailureHandler;
         this.threadContext = threadPool.getThreadContext();
         this.anonymousUser = anonymousUser;

plugin/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizedIndices.java

@@ -9,12 +9,9 @@ import org.elasticsearch.cluster.metadata.AliasOrIndex;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.authz.permission.Role;
-import org.elasticsearch.xpack.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.security.user.User;
-import org.elasticsearch.xpack.security.user.XPackUser;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;

plugin/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java

@@ -11,22 +11,32 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
+import org.elasticsearch.transport.RemoteClusterAware;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.metadata.AliasOrIndex;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.transport.TransportRequest;
 
+import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.SortedMap;
+import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.stream.Collectors;
 
 public class IndicesAndAliasesResolver {
@@ -39,9 +49,11 @@ public class IndicesAndAliasesResolver {
     static final List<String> NO_INDICES_LIST = Arrays.asList(NO_INDICES_ARRAY);
 
     private final IndexNameExpressionResolver nameExpressionResolver;
+    private final RemoteClusterResolver remoteClusterResolver;
 
-    public IndicesAndAliasesResolver(IndexNameExpressionResolver nameExpressionResolver) {
-        this.nameExpressionResolver = nameExpressionResolver;
+    public IndicesAndAliasesResolver(Settings settings, ClusterService clusterService) {
+        this.nameExpressionResolver = new IndexNameExpressionResolver(settings);
+        this.remoteClusterResolver = new RemoteClusterResolver(settings, clusterService.getClusterSettings());
     }
 
     public Set<String> resolve(TransportRequest request, MetaData metaData, AuthorizedIndices authorizedIndices) {
@@ -98,13 +110,21 @@ public class IndicesAndAliasesResolver {
                 // if we cannot replace wildcards the indices list stays empty. Same if there are no authorized indices.
                 // we honour allow_no_indices like es core does.
             } else {
-                replacedIndices = replaceWildcardsWithAuthorizedIndices(indicesRequest.indices(),
-                        indicesOptions, metaData, authorizedIndices.get(), replaceWildcards);
+                String[] localIndexNames = indicesRequest.indices();
+                List<String> remoteIndices = Collections.emptyList();
+                if (allowsRemoteIndices(indicesRequest)) {
+                    final Tuple<List<String>, List<String>> split = remoteClusterResolver.splitLocalAndRemoteIndexNames(localIndexNames);
+                    localIndexNames = split.v1().toArray(new String[split.v1().size()]);
+                    remoteIndices = split.v2();
+                }
+                replacedIndices = replaceWildcardsWithAuthorizedIndices(localIndexNames, indicesOptions, metaData, authorizedIndices.get(),
+                        replaceWildcards);
                 if (indicesOptions.ignoreUnavailable()) {
                     //out of all the explicit names (expanded from wildcards and original ones that were left untouched)
                     //remove all the ones that the current user is not authorized for and ignore them
                     replacedIndices = replacedIndices.stream().filter(authorizedIndices.get()::contains).collect(Collectors.toList());
                 }
+                replacedIndices.addAll(remoteIndices);
             }
             if (replacedIndices.isEmpty()) {
                 if (indicesOptions.allowNoIndices()) {
@@ -164,6 +184,10 @@ public class IndicesAndAliasesResolver {
         return Collections.unmodifiableSet(indices);
     }
 
+    private static boolean allowsRemoteIndices(IndicesRequest indicesRequest) {
+        return indicesRequest instanceof SearchRequest || indicesRequest instanceof FieldCapabilitiesRequest;
+    }
+
     private List<String> loadAuthorizedAliases(List<String> authorizedIndices, MetaData metaData) {
         List<String> authorizedAliases = new ArrayList<>();
         SortedMap<String, AliasOrIndex> existingAliases = metaData.getAliasAndIndexLookup();
@@ -331,4 +355,39 @@ public class IndicesAndAliasesResolver {
     private static List<String> indicesList(String[] list) {
         return (list == null) ? null : Arrays.asList(list);
     }
+
+    private static class RemoteClusterResolver extends RemoteClusterAware {
+
+        private final CopyOnWriteArraySet<String> clusters;
+
+        private RemoteClusterResolver(Settings settings, ClusterSettings clusterSettings) {
+            super(settings);
+            clusters = new CopyOnWriteArraySet<>(buildRemoteClustersSeeds(settings).keySet());
+            listenForUpdates(clusterSettings);
+        }
+
+        @Override
+        protected Set<String> getRemoteClusterNames() {
+            return clusters;
+        }
+
+        @Override
+        protected void updateRemoteCluster(String clusterAlias, List<InetSocketAddress> addresses) {
+            if (addresses.isEmpty()) {
+                clusters.remove(clusterAlias);
+            } else {
+                clusters.add(clusterAlias);
+            }
+        }
+
+        Tuple<List<String>, List<String>> splitLocalAndRemoteIndexNames(String ... indices) {
+            final Map<String, List<String>> map = super.groupClusterIndices(indices, exists -> false);
+            final List<String> local = map.remove(LOCAL_CLUSTER_GROUP_KEY);
+            final List<String> remote = map.entrySet().stream()
+                    .flatMap(e -> e.getValue().stream().map(v -> e.getKey() + REMOTE_CLUSTER_INDEX_SEPARATOR + v))
+                    .collect(Collectors.toList());
+            return new Tuple<>(local == null ? Collections.emptyList() : local, remote);
+        }
+
+    }
 }

plugin/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java

@@ -40,6 +40,7 @@ import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusAction;
 import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusRequest;
 import org.elasticsearch.action.bulk.BulkAction;
 import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.index.reindex.DeleteByQueryAction;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
 import org.elasticsearch.action.delete.DeleteAction;
@@ -151,6 +152,8 @@ public class AuthorizationServiceTests extends ESTestCase {
     public void setup() {
         rolesStore = mock(CompositeRolesStore.class);
         clusterService = mock(ClusterService.class);
+        final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); 
+        when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
         auditTrail = mock(AuditTrailService.class);
         threadContext = new ThreadContext(Settings.EMPTY);
         threadPool = mock(ThreadPool.class);

plugin/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java

@@ -13,11 +13,17 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.action.admin.indices.close.CloseIndexAction;
+import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
 import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesAction;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
 import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.search.MultiSearchAction;
 import org.elasticsearch.action.search.MultiSearchRequest;
@@ -32,7 +38,9 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.search.internal.ShardSearchTransportRequest;
@@ -58,7 +66,9 @@ import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasItems;
@@ -86,6 +96,8 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 2))
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, 1))
+                .put("search.remote.remote.seeds", "127.0.0.1:" + randomIntBetween(9301, 9350))
+                .put("search.remote.other_remote.seeds", "127.0.0.1:" + randomIntBetween(9351, 9399))
                 .build();
 
         indexNameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY);
@@ -144,10 +156,11 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
             }).when(rolesStore).roles(any(Set.class), any(FieldPermissionsCache.class), any(ActionListener.class));
 
         ClusterService clusterService = mock(ClusterService.class);
+        when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
         authzService = new AuthorizationService(settings, rolesStore, clusterService,
                 mock(AuditTrailService.class), new DefaultAuthenticationFailureHandler(), mock(ThreadPool.class),
                 new AnonymousUser(settings));
-        defaultIndicesResolver = new IndicesAndAliasesResolver(indexNameExpressionResolver);
+        defaultIndicesResolver = new IndicesAndAliasesResolver(settings, clusterService);
     }
 
     public void testDashIndicesAreAllowedInShardLevelRequests() {
@@ -506,6 +519,31 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         assertNoIndices(request, defaultIndicesResolver.resolve(request, metaData, buildAuthorizedIndices(user, SearchAction.NAME)));
     }
 
+    public void testSearchWithRemoteIndex() {
+        SearchRequest request = new SearchRequest("remote:indexName");
+        request.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
+        final Set<String> resolved = defaultIndicesResolver.resolve(request, metaData, buildAuthorizedIndices(user, SearchAction.NAME));
+        assertThat(resolved, containsInAnyOrder("remote:indexName"));
+        assertThat(request.indices(), arrayContaining("remote:indexName"));
+    }
+
+    public void testSearchWithRemoteAndLocalIndices() {
+        SearchRequest request = new SearchRequest("remote:indexName", "bar", "bar2");
+        request.indicesOptions(IndicesOptions.fromOptions(true, randomBoolean(), randomBoolean(), randomBoolean()));
+        final Set<String> resolved = defaultIndicesResolver.resolve(request, metaData, buildAuthorizedIndices(user, SearchAction.NAME));
+        assertThat(resolved, containsInAnyOrder("remote:indexName", "bar"));
+        assertThat(request.indices(), arrayContainingInAnyOrder("remote:indexName", "bar"));
+    }
+
+    public void testSearchWithRemoteAndLocalWildcards() {
+        SearchRequest request = new SearchRequest("*:foo", "r*:bar*", "remote:baz*", "bar*", "foofoo");
+        request.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false));
+        final Set<String> resolved = defaultIndicesResolver.resolve(request, metaData, buildAuthorizedIndices(user, SearchAction.NAME));
+        final String[] expectedIndices = { "remote:foo", "other_remote:foo", "remote:bar*", "remote:baz*", "bar", "foofoo" };
+        assertThat(resolved, containsInAnyOrder(expectedIndices));
+        assertThat(request.indices(), arrayContainingInAnyOrder(expectedIndices));
+    }
+
     public void testResolveIndicesAliasesRequest() {
         IndicesAliasesRequest request = new IndicesAliasesRequest();
         request.addAliasAction(AliasActions.add().alias("alias1").indices("foo", "foofoo"));
@@ -1011,6 +1049,46 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         assertEquals("no such index", e.getMessage());
     }
 
+    /**
+     * Tests that all the request types that are known to support remote indices successfully pass them through
+     *  the resolver
+     */
+    public void testRemotableRequestsAllowRemoteIndices() {
+        IndicesOptions options = IndicesOptions.fromOptions(true, false, false, false);
+        Tuple<TransportRequest, String> tuple = randomFrom(
+                new Tuple<>(new SearchRequest("remote:foo").indicesOptions(options), SearchAction.NAME),
+                new Tuple<>(new FieldCapabilitiesRequest().indices("remote:foo").indicesOptions(options), FieldCapabilitiesAction.NAME)
+        );
+        final Set<String> resolved = defaultIndicesResolver.resolve(tuple.v1(), metaData, buildAuthorizedIndices(user, tuple.v2()));
+        assertThat(resolved, containsInAnyOrder("remote:foo"));
+    }
+
+    /**
+     * Tests that request types that do not support remote indices will be resolved as if all index names are local.
+     */
+    public void testNonRemotableRequestDoesNotAllowRemoteIndices() {
+        IndicesOptions options = IndicesOptions.fromOptions(true, false, false, false);
+        Tuple<TransportRequest, String> tuple = randomFrom(
+                new Tuple<>(new CloseIndexRequest("remote:foo").indicesOptions(options), CloseIndexAction.NAME),
+                new Tuple<>(new DeleteIndexRequest("remote:foo").indicesOptions(options), DeleteIndexAction.NAME),
+                new Tuple<>(new PutMappingRequest("remote:foo").indicesOptions(options), PutMappingAction.NAME)
+        );
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class,
+                () -> defaultIndicesResolver.resolve(tuple.v1(), metaData, buildAuthorizedIndices(user, tuple.v2())));
+        assertEquals("no such index", e.getMessage());
+    }
+
+    public void testNonRemotableRequestDoesNotAllowRemoteWildcardIndices() {
+        IndicesOptions options = IndicesOptions.fromOptions(randomBoolean(), true, true, true);
+        Tuple<TransportRequest, String> tuple = randomFrom(
+                new Tuple<>(new CloseIndexRequest("*:*").indicesOptions(options), CloseIndexAction.NAME),
+                new Tuple<>(new DeleteIndexRequest("*:*").indicesOptions(options), DeleteIndexAction.NAME),
+                new Tuple<>(new PutMappingRequest("*:*").indicesOptions(options), PutMappingAction.NAME)
+        );
+        final Set<String> resolved = defaultIndicesResolver.resolve(tuple.v1(), metaData, buildAuthorizedIndices(user, tuple.v2()));
+        assertNoIndices((IndicesRequest.Replaceable) tuple.v1(), resolved);
+    }
+
     public void testCompositeIndicesRequestIsNotSupported() {
         TransportRequest request = randomFrom(new MultiSearchRequest(), new MultiGetRequest(),
                 new MultiTermVectorsRequest(), new BulkRequest());

qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yaml

@@ -22,7 +22,7 @@ setup:
               "cluster": ["all"],
               "indices": [
                 {
-                  "names": ["test_index", "my_remote_cluster:test_index", "my_remote_cluster:aliased_test_index", "test_remote_cluster:test_index", "my_remote_cluster:secure_alias"],
+                  "names": ["local_index", "my_remote_cluster:test_i*", "my_remote_cluster:aliased_test_index", "test_remote_cluster:test_i*", "my_remote_cluster:secure_alias"],
                   "privileges": ["read"]
                 }
               ]
@@ -42,7 +42,7 @@ teardown:
 
   - do:
       indices.create:
-        index: test_index
+        index: local_index
         body:
           settings:
             index:
@@ -53,21 +53,21 @@ teardown:
       bulk:
         refresh: true
         body:
-            - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+            - '{"index": {"_index": "local_index", "_type": "test_type"}}'
             - '{"f1": "local_cluster", "filter_field": 0}'
-            - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+            - '{"index": {"_index": "local_index", "_type": "test_type"}}'
             - '{"f1": "local_cluster", "filter_field": 1}'
-            - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+            - '{"index": {"_index": "local_index", "_type": "test_type"}}'
             - '{"f1": "local_cluster", "filter_field": 0}'
-            - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+            - '{"index": {"_index": "local_index", "_type": "test_type"}}'
             - '{"f1": "local_cluster", "filter_field": 1}'
-            - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+            - '{"index": {"_index": "local_index", "_type": "test_type"}}'
             - '{"f1": "local_cluster", "filter_field": 0}'
 
   - do:
       headers: { Authorization: "Basic am9lOnMza3JpdA==" }
       search:
-        index: test_index,my_remote_cluster:test_index
+        index: local_index,my_remote_cluster:test_index
         body:
           aggs:
             cluster:
@@ -85,7 +85,7 @@ teardown:
   - do:
       headers: { Authorization: "Basic am9lOnMza3JpdA==" }
       search:
-        index: test_index,my_remote_cluster:test_index
+        index: local_index,my_remote_cluster:test_index
         body:
           query:
             term:
@@ -119,10 +119,28 @@ teardown:
   - match: { aggregations.cluster.buckets.0.key: "remote_cluster" }
   - match: { aggregations.cluster.buckets.0.doc_count: 6 }
 
+  # Test wildcard in cluster name
   - do:
       headers: { Authorization: "Basic am9lOnMza3JpdA==" }
       search:
-        index: test_index
+        index: "my_*:test_index"
+        body:
+          aggs:
+            cluster:
+              terms:
+                field: f1.keyword
+
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 6}
+  - match: { hits.hits.0._index: "my_remote_cluster:test_index"}
+  - length: { aggregations.cluster.buckets: 1 }
+  - match: { aggregations.cluster.buckets.0.key: "remote_cluster" }
+  - match: { aggregations.cluster.buckets.0.doc_count: 6 }
+
+  - do:
+      headers: { Authorization: "Basic am9lOnMza3JpdA==" }
+      search:
+        index: local_index
         body:
           aggs:
             cluster:
@@ -131,7 +149,7 @@ teardown:
 
   - match: { _shards.total: 2 }
   - match: { hits.total: 5}
-  - match: { hits.hits.0._index: "test_index"}
+  - match: { hits.hits.0._index: "local_index"}
   - length: { aggregations.cluster.buckets: 1 }
   - match: { aggregations.cluster.buckets.0.key: "local_cluster" }
   - match: { aggregations.cluster.buckets.0.doc_count: 5 }
@@ -162,6 +180,15 @@ teardown:
   - match: { hits.total: 6 }
   - match: { hits.hits.0._index: "test_remote_cluster:test_index" }
 
+  # Test wildcard that matches multiple (two) cluster names
+  - do:
+      headers: { Authorization: "Basic am9lOnMza3JpdA==" }
+      search:
+        index: "*_remote_cluster:test_ind*"
+
+  - match: { _shards.total: 6 }
+  - match: { hits.total: 12 }
+
 ---
 "Search an filtered alias on the remote cluster":