honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[7.x] Mirror privileges over data streams to their backing indices (#58991)

This commit is contained in:
Dan Hermann 2020-07-03 06:33:38 -05:00 committed by GitHub
parent 4f86f6fb38
commit 5e7746d3bd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 634 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.idea/runConfigurations/Debug_Elasticsearch.xml
View File

@ -8,4 +8,5 @@
<option name="AUTO_RESTART" value="true" />
<method v="2" />
</configuration>
</component>
</component>

9
rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml
View File

@ -45,15 +45,6 @@
indices.delete:
index: logs-foobar
- do:
indices.create:
index: logs-foobarbaz
- do:
catch: bad_request
indices.close:
index: logs-*
- do:
indices.delete_data_stream:
name: logs-foobar

9
server/src/main/java/org/elasticsearch/action/IndicesRequest.java
View File

@ -40,6 +40,15 @@ public interface IndicesRequest {
*/
IndicesOptions indicesOptions();
/**
* Determines whether the request should be applied to data streams. When {@code false}, none of the names or
* wildcard expressions in {@link #indices} should be applied to or expanded to any data streams. All layers
* involved in the request's fulfillment including security, name resolution, etc., should respect this flag.
*/
default boolean includeDataStreams() {
return false;
}
interface Replaceable extends IndicesRequest {
/**
* Sets the indices that the action relates to.

5
server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java
View File

@ -112,6 +112,11 @@ public class ClusterSearchShardsRequest extends MasterNodeReadRequest<ClusterSea
return this;
}
@Override
public boolean includeDataStreams() {
return true;
}
/**
* A comma separated list of routing values to control the shards the search will be executed on.
*/

5
server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java
View File

@ -290,4 +290,9 @@ public class IndicesStatsRequest extends BroadcastRequest<IndicesStatsRequest> {
super.writeTo(out);
flags.writeTo(out);
}
@Override
public boolean includeDataStreams() {
return true;
}
}

5
server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java
View File

@ -157,6 +157,11 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
return indicesOptions;
}
@Override
public boolean includeDataStreams() {
return true;
}
public boolean includeUnmapped() {
return includeUnmapped;
}

5
server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
View File

@ -356,6 +356,11 @@ public class SearchRequest extends ActionRequest implements IndicesRequest.Repla
return this;
}
@Override
public boolean includeDataStreams() {
return true;
}
/**
* Returns whether network round-trips should be minimized when executing cross-cluster search requests.
* Defaults to <code>true</code>.

11
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java
View File

@ -194,7 +194,7 @@ public final class IndicesPermission {
* Authorizes the provided action against the provided indices, given the current cluster metadata
*/
public Map<String, IndicesAccessControl.IndexAccessControl> authorize(String action, Set<String> requestedIndicesOrAliases,
Map<String, IndexAbstraction> allAliasesAndIndices,
Map<String, IndexAbstraction> lookup,
FieldPermissionsCache fieldPermissionsCache) {
// now... every index that is associated with the request, must be granted
// by at least one indices permission group
@ -205,7 +205,7 @@ public final class IndicesPermission {
for (String indexOrAlias : requestedIndicesOrAliases) {
boolean granted = false;
Set<String> concreteIndices = new HashSet<>();
IndexAbstraction indexAbstraction = allAliasesAndIndices.get(indexOrAlias);
IndexAbstraction indexAbstraction = lookup.get(indexOrAlias);
if (indexAbstraction != null) {
for (IndexMetadata indexMetadata : indexAbstraction.getIndices()) {
concreteIndices.add(indexMetadata.getIndex().getName());
@ -213,7 +213,12 @@ public final class IndicesPermission {
}
for (Group group : groups) {
if (group.check(action, indexOrAlias)) {
// check for privilege granted directly on the requested index/alias
if (group.check(action, indexOrAlias) ||
// check for privilege granted on parent data stream if a backing index
(indexAbstraction != null && indexAbstraction.getType() == IndexAbstraction.Type.CONCRETE_INDEX &&
indexAbstraction.getParentDataStream() != null &&
group.check(action, indexAbstraction.getParentDataStream().getName()))) {
granted = true;
for (String index : concreteIndices) {
Set<FieldPermissions> fieldPermissions = fieldPermissionsByIndex.computeIfAbsent(index, (k) -> new HashSet<>());

28
x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
View File

@ -137,7 +137,7 @@ class IndicesAndAliasesResolver {
if (IndexNameExpressionResolver.isAllIndices(indicesList(indicesRequest.indices()))) {
if (replaceWildcards) {
for (String authorizedIndex : authorizedIndices) {
if (isIndexVisible("*", authorizedIndex, indicesOptions, metadata)) {
if (isIndexVisible("*", authorizedIndex, indicesOptions, metadata, indicesRequest.includeDataStreams())) {
resolvedIndicesBuilder.addLocal(authorizedIndex);
}
}
@ -152,7 +152,7 @@ class IndicesAndAliasesResolver {
split = new ResolvedIndices(Arrays.asList(indicesRequest.indices()), Collections.emptyList());
}
List<String> replaced = replaceWildcardsWithAuthorizedIndices(split.getLocal(), indicesOptions, metadata,
authorizedIndices, replaceWildcards);
authorizedIndices, replaceWildcards, indicesRequest.includeDataStreams());
if (indicesOptions.ignoreUnavailable()) {
//out of all the explicit names (expanded from wildcards and original ones that were left untouched)
//remove all the ones that the current user is not authorized for and ignore them
@ -344,7 +344,8 @@ class IndicesAndAliasesResolver {
//TODO Investigate reusing code from vanilla es to resolve index names and wildcards
private List<String> replaceWildcardsWithAuthorizedIndices(Iterable<String> indices, IndicesOptions indicesOptions, Metadata metadata,
List<String> authorizedIndices, boolean replaceWildcards) {
List<String> authorizedIndices, boolean replaceWildcards,
boolean includeDataStreams) {
//the order matters when it comes to exclusions
List<String> finalIndices = new ArrayList<>();
boolean wildcardSeen = false;
@ -366,7 +367,7 @@ class IndicesAndAliasesResolver {
// continue
aliasOrIndex = dateMathName;
} else if (authorizedIndices.contains(dateMathName) &&
isIndexVisible(aliasOrIndex, dateMathName, indicesOptions, metadata, true)) {
isIndexVisible(aliasOrIndex, dateMathName, indicesOptions, metadata, includeDataStreams, true)) {
if (minus) {
finalIndices.remove(dateMathName);
} else {
@ -384,7 +385,7 @@ class IndicesAndAliasesResolver {
Set<String> resolvedIndices = new HashSet<>();
for (String authorizedIndex : authorizedIndices) {
if (Regex.simpleMatch(aliasOrIndex, authorizedIndex) &&
isIndexVisible(aliasOrIndex, authorizedIndex, indicesOptions, metadata)) {
isIndexVisible(aliasOrIndex, authorizedIndex, indicesOptions, metadata, includeDataStreams)) {
resolvedIndices.add(authorizedIndex);
}
}
@ -419,13 +420,17 @@ class IndicesAndAliasesResolver {
return finalIndices;
}
private static boolean isIndexVisible(String expression, String index, IndicesOptions indicesOptions, Metadata metadata) {
return isIndexVisible(expression, index, indicesOptions, metadata, false);
private static boolean isIndexVisible(String expression, String index, IndicesOptions indicesOptions, Metadata metadata,
boolean includeDataStreams) {
return isIndexVisible(expression, index, indicesOptions, metadata, includeDataStreams, false);
}
private static boolean isIndexVisible(String expression, String index, IndicesOptions indicesOptions, Metadata metadata,
boolean dateMathExpression) {
boolean includeDataStreams, boolean dateMathExpression) {
IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(index);
if (indexAbstraction == null) {
throw new IllegalStateException("could not resolve index abstraction [" + index + "]");
}
final boolean isHidden = indexAbstraction.isHidden();
if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS) {
//it's an alias, ignore expandWildcardsOpen and expandWildcardsClosed.
@ -440,12 +445,7 @@ class IndicesAndAliasesResolver {
}
}
if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) {
// If indicesOptions.includeDataStreams() returns false then we fail later in IndexNameExpressionResolver.
if (isHidden == false || indicesOptions.expandWildcardsHidden()) {
return true;
} else {
return false;
}
return includeDataStreams;
}
assert indexAbstraction.getIndices().size() == 1 : "concrete index must point to a single index";
IndexMetadata indexMetadata = indexAbstraction.getIndices().get(0);

30
x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
View File

@ -86,6 +86,7 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import static org.elasticsearch.common.Strings.arrayToCommaDelimitedString;
import static org.elasticsearch.xpack.security.action.user.TransportHasPrivilegesAction.getApplicationNames;
@ -335,7 +336,7 @@ public class RBACEngine implements AuthorizationEngine {
Map<String, IndexAbstraction> indicesLookup, ActionListener<List<String>> listener) {
if (authorizationInfo instanceof RBACAuthorizationInfo) {
final Role role = ((RBACAuthorizationInfo) authorizationInfo).getRole();
listener.onResponse(resolveAuthorizedIndicesFromRole(role, requestInfo.getAction(), indicesLookup));
listener.onResponse(resolveAuthorizedIndicesFromRole(role, requestInfo, indicesLookup));
} else {
listener.onFailure(
new IllegalArgumentException("unsupported authorization info:" + authorizationInfo.getClass().getSimpleName()));
@ -492,18 +493,29 @@ public class RBACEngine implements AuthorizationEngine {
return new GetUserPrivilegesResponse(cluster, conditionalCluster, indices, application, runAs);
}
static List<String> resolveAuthorizedIndicesFromRole(Role role, String action, Map<String, IndexAbstraction> aliasAndIndexLookup) {
Predicate<String> predicate = role.allowedIndicesMatcher(action);
static List<String> resolveAuthorizedIndicesFromRole(Role role, RequestInfo requestInfo, Map<String, IndexAbstraction> lookup) {
Predicate<String> predicate = role.allowedIndicesMatcher(requestInfo.getAction());
List<String> indicesAndAliases = new ArrayList<>();
// do not include data streams for actions that do not operate on data streams
TransportRequest request = requestInfo.getRequest();
boolean includeDataStreams = (request instanceof IndicesRequest) && ((IndicesRequest) request).includeDataStreams();
Set<String> indicesAndAliases = new HashSet<>();
// TODO: can this be done smarter? I think there are usually more indices/aliases in the cluster then indices defined a roles?
for (Map.Entry<String, IndexAbstraction> entry : aliasAndIndexLookup.entrySet()) {
String aliasOrIndex = entry.getKey();
if (predicate.test(aliasOrIndex)) {
indicesAndAliases.add(aliasOrIndex);
for (Map.Entry<String, IndexAbstraction> entry : lookup.entrySet()) {
String indexAbstraction = entry.getKey();
if (predicate.test(indexAbstraction)) {
if (entry.getValue().getType() != IndexAbstraction.Type.DATA_STREAM) {
indicesAndAliases.add(indexAbstraction);
} else if (includeDataStreams) {
// add data stream and its backing indices for any authorized data streams
indicesAndAliases.addAll(entry.getValue().getIndices().stream()
.map(i -> i.getIndex().getName()).collect(Collectors.toList()));
indicesAndAliases.add(indexAbstraction);
}
}
}
return Collections.unmodifiableList(indicesAndAliases);
return Collections.unmodifiableList(new ArrayList<>(indicesAndAliases));
}
private void buildIndicesAccessControl(Authentication authentication, String action,

32
x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java
View File

@ -14,6 +14,8 @@ import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.transport.TransportRequest;
import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
@ -36,7 +38,7 @@ public class AuthorizedIndicesTests extends ESTestCase {
public void testAuthorizedIndicesUserWithoutRoles() {
List<String> authorizedIndices =
RBACEngine.resolveAuthorizedIndicesFromRole(Role.EMPTY, "", Metadata.EMPTY_METADATA.getIndicesLookup());
RBACEngine.resolveAuthorizedIndicesFromRole(Role.EMPTY, getRequestInfo(""), Metadata.EMPTY_METADATA.getIndicesLookup());
assertTrue(authorizedIndices.isEmpty());
}
@ -72,7 +74,7 @@ public class AuthorizedIndicesTests extends ESTestCase {
CompositeRolesStore.buildRoleFromDescriptors(descriptors, new FieldPermissionsCache(Settings.EMPTY), null, future);
Role roles = future.actionGet();
List<String> list =
RBACEngine.resolveAuthorizedIndicesFromRole(roles, SearchAction.NAME, metadata.getIndicesLookup());
RBACEngine.resolveAuthorizedIndicesFromRole(roles, getRequestInfo(SearchAction.NAME), metadata.getIndicesLookup());
assertThat(list, containsInAnyOrder("a1", "a2", "aaaaaa", "b", "ab"));
assertFalse(list.contains("bbbbb"));
assertFalse(list.contains("ba"));
@ -82,16 +84,18 @@ public class AuthorizedIndicesTests extends ESTestCase {
public void testAuthorizedIndicesUserWithSomeRolesEmptyMetadata() {
Role role = Role.builder("role").add(IndexPrivilege.ALL, "*").build();
List<String> authorizedIndices =
RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, Metadata.EMPTY_METADATA.getIndicesLookup());
List<String> authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(role, getRequestInfo(SearchAction.NAME),
Metadata.EMPTY_METADATA.getIndicesLookup());
assertTrue(authorizedIndices.isEmpty());
}
public void testSecurityIndicesAreRemovedFromRegularUser() {
Role role = Role.builder("user_role").add(IndexPrivilege.ALL, "*").cluster(Collections.singleton("all"), Collections.emptySet())
Role role = Role.builder("user_role").add(IndexPrivilege.ALL, "*").cluster(
org.elasticsearch.common.collect.Set.of("all"),
org.elasticsearch.common.collect.Set.of())
.build();
List<String> authorizedIndices =
RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, Metadata.EMPTY_METADATA.getIndicesLookup());
List<String> authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(role, getRequestInfo(SearchAction.NAME),
Metadata.EMPTY_METADATA.getIndicesLookup());
assertTrue(authorizedIndices.isEmpty());
}
@ -116,7 +120,7 @@ public class AuthorizedIndicesTests extends ESTestCase {
.build();
List<String> authorizedIndices =
RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, metadata.getIndicesLookup());
RBACEngine.resolveAuthorizedIndicesFromRole(role, getRequestInfo(SearchAction.NAME), metadata.getIndicesLookup());
assertThat(authorizedIndices, containsInAnyOrder("an-index", "another-index"));
assertThat(authorizedIndices, not(contains(internalSecurityIndex)));
assertThat(authorizedIndices, not(contains(RestrictedIndicesNames.SECURITY_MAIN_ALIAS)));
@ -142,13 +146,21 @@ public class AuthorizedIndicesTests extends ESTestCase {
.build();
List<String> authorizedIndices =
RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, metadata.getIndicesLookup());
RBACEngine.resolveAuthorizedIndicesFromRole(role, getRequestInfo(SearchAction.NAME), metadata.getIndicesLookup());
assertThat(authorizedIndices, containsInAnyOrder(
"an-index", "another-index", RestrictedIndicesNames.SECURITY_MAIN_ALIAS, internalSecurityIndex));
List<String> authorizedIndicesSuperUser =
RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, metadata.getIndicesLookup());
RBACEngine.resolveAuthorizedIndicesFromRole(role, getRequestInfo(SearchAction.NAME), metadata.getIndicesLookup());
assertThat(authorizedIndicesSuperUser, containsInAnyOrder(
"an-index", "another-index", RestrictedIndicesNames.SECURITY_MAIN_ALIAS, internalSecurityIndex));
}
public static AuthorizationEngine.RequestInfo getRequestInfo(String action) {
return getRequestInfo(TransportRequest.Empty.INSTANCE, action);
}
public static AuthorizationEngine.RequestInfo getRequestInfo(TransportRequest request, String action) {
return new AuthorizationEngine.RequestInfo(null, request, action);
}
}

303
x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
View File

@ -83,6 +83,9 @@ import java.util.Set;
import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField;
import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;
import static org.elasticsearch.xpack.security.authz.AuthorizedIndicesTests.getRequestInfo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.Matchers.arrayContaining;
import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
import static org.hamcrest.Matchers.contains;
@ -217,6 +220,27 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
.privileges("all")
.build()
}, null));
roleMap.put("data_stream_test3", new RoleDescriptor("data_stream_test3", null,
new IndicesPrivileges[] {
IndicesPrivileges.builder()
.indices("logs*")
.privileges("all")
.build()
}, null));
roleMap.put("backing_index_test_wildcards", new RoleDescriptor("backing_index_test_wildcards", null,
new IndicesPrivileges[] {
IndicesPrivileges.builder()
.indices(".ds-logs*")
.privileges("all")
.build()
}, null));
roleMap.put("backing_index_test_name", new RoleDescriptor("backing_index_test_name", null,
new IndicesPrivileges[] {
IndicesPrivileges.builder()
.indices(dataStreamIndex1.getIndex().getName())
.privileges("all")
.build()
}, null));
final FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
doAnswer((i) -> {
ActionListener callback =
@ -1554,13 +1578,13 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
public void testDataStreamResolution() {
{
final User user = new User("data-steam-tester1", "data_stream_test1");
final List<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
final User user = new User("data-stream-tester1", "data_stream_test1");
// Resolve data streams:
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("logs-*");
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, false, true, true, true, true));
final List<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME, searchRequest);
ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), contains("logs-foobar"));
assertThat(resolvedIndices.getRemote(), emptyIterable());
@ -1575,25 +1599,294 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
assertThat(resolvedIndices.getRemote(), emptyIterable());
}
{
final User user = new User("data-steam-tester2", "data_stream_test2");
final List<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
final User user = new User("data-stream-tester2", "data_stream_test2");
// Resolve *all* data streams:
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("logs-*");
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, false, true, true, true, true));
final List<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME, searchRequest);
ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), containsInAnyOrder("logs-foo", "logs-foobar"));
assertThat(resolvedIndices.getRemote(), emptyIterable());
}
}
public void testDataStreamsAreNotVisibleWhenNotIncludedByRequestWithWildcard() {
final User user = new User("data-stream-tester2", "data_stream_test2");
GetAliasesRequest request = new GetAliasesRequest("*");
assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
assertThat(request.includeDataStreams(), is(false));
// data streams and their backing indices should _not_ be in the authorized list since the backing indices
// do not match the requested pattern
List<String> dataStreams = org.elasticsearch.common.collect.List.of("logs-foo", "logs-foobar");
final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
for (String dsName : dataStreams) {
assertThat(authorizedIndices, not(hasItem(dsName)));
DataStream dataStream = metadata.dataStreams().get(dsName);
assertThat(authorizedIndices, not(hasItem(dsName)));
for (Index i : dataStream.getIndices()) {
assertThat(authorizedIndices, not(hasItem(i.getName())));
}
}
// neither data streams nor their backing indices will be in the resolved list unless the backing indices matched the requested
// pattern
ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(request, metadata, authorizedIndices);
for (String dsName : dataStreams) {
assertThat(resolvedIndices.getLocal(), not(hasItem(dsName)));
DataStream dataStream = metadata.dataStreams().get(dsName);
assertThat(resolvedIndices.getLocal(), not(hasItem(dsName)));
for (Index i : dataStream.getIndices()) {
assertThat(resolvedIndices.getLocal(), not(hasItem(i.getName())));
}
}
}
public void testDataStreamsAreNotVisibleWhenNotIncludedByRequestWithoutWildcard() {
final User user = new User("data-stream-tester2", "data_stream_test2");
String dataStreamName = "logs-foobar";
GetAliasesRequest request = new GetAliasesRequest(dataStreamName);
assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
assertThat(request.includeDataStreams(), is(false));
// data streams and their backing indices should _not_ be in the authorized list since the backing indices
// do not match the requested name
final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
assertThat(authorizedIndices, not(hasItem(dataStreamName)));
DataStream dataStream = metadata.dataStreams().get(dataStreamName);
assertThat(authorizedIndices, not(hasItem(dataStreamName)));
for (Index i : dataStream.getIndices()) {
assertThat(authorizedIndices, not(hasItem(i.getName())));
}
// neither data streams nor their backing indices will be in the resolved list since the backing indices do not match the
// requested name(s)
ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(request, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), not(hasItem(dataStreamName)));
for (Index i : dataStream.getIndices()) {
assertThat(resolvedIndices.getLocal(), not(hasItem(i.getName())));
}
}
public void testDataStreamsAreVisibleWhenIncludedByRequestWithWildcard() {
final User user = new User("data-stream-tester3", "data_stream_test3");
SearchRequest request = new SearchRequest("logs*");
assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
assertThat(request.includeDataStreams(), is(true));
// data streams and their backing indices should be in the authorized list
List<String> expectedDataStreams = org.elasticsearch.common.collect.List.of("logs-foo", "logs-foobar");
final List<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME, request);
for (String dsName : expectedDataStreams) {
DataStream dataStream = metadata.dataStreams().get(dsName);
assertThat(authorizedIndices, hasItem(dsName));
for (Index i : dataStream.getIndices()) {
assertThat(authorizedIndices, hasItem(i.getName()));
}
}
// data streams without their backing indices will be in the resolved list since the backing indices do not match the requested
// pattern
ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(request, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), hasItem("logs-foo"));
assertThat(resolvedIndices.getLocal(), hasItem("logs-foobar"));
assertThat(resolvedIndices.getLocal(), hasItem("logs-00001"));
assertThat(resolvedIndices.getLocal(), hasItem("logs-00002"));
assertThat(resolvedIndices.getLocal(), hasItem("logs-00003"));
assertThat(resolvedIndices.getLocal(), hasItem("logs-alias"));
for (String dsName : expectedDataStreams) {
DataStream dataStream = metadata.dataStreams().get(dsName);
assertNotNull(dataStream);
for (Index i : dataStream.getIndices()) {
assertThat(resolvedIndices.getLocal(), not(hasItem(i.getName())));
}
}
}
public void testDataStreamsAreVisibleWhenIncludedByRequestWithoutWildcard() {
final User user = new User("data-stream-tester3", "data_stream_test3");
String dataStreamName = "logs-foobar";
DataStream dataStream = metadata.dataStreams().get(dataStreamName);
SearchRequest request = new SearchRequest(dataStreamName);
assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
assertThat(request.includeDataStreams(), is(true));
final List<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME, request);
// data streams and their backing indices should be in the authorized list
assertThat(authorizedIndices, hasItem(dataStreamName));
for (Index i : dataStream.getIndices()) {
assertThat(authorizedIndices, hasItem(i.getName()));
}
ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(request, metadata, authorizedIndices);
// data streams without their backing indices will be in the resolved list since the backing indices do not match the requested
// name
assertThat(resolvedIndices.getLocal(), hasItem(dataStreamName));
for (Index i : dataStream.getIndices()) {
assertThat(resolvedIndices.getLocal(), not(hasItem(i.getName())));
}
}
public void testBackingIndicesAreVisibleWhenIncludedByRequestWithWildcard() {
final User user = new User("data-stream-tester3", "data_stream_test3");
SearchRequest request = new SearchRequest(".ds-logs*");
assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
assertThat(request.includeDataStreams(), is(true));
// data streams and their backing indices should be included in the authorized list
List<String> expectedDataStreams = org.elasticsearch.common.collect.List.of("logs-foo", "logs-foobar");
final List<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME, request);
for (String dsName : expectedDataStreams) {
DataStream dataStream = metadata.dataStreams().get(dsName);
assertThat(authorizedIndices, hasItem(dsName));
for (Index i : dataStream.getIndices()) {
assertThat(authorizedIndices, hasItem(i.getName()));
}
}
// data streams should _not_ be included in the resolved list because they do not match the pattern but their backing indices
// should be in the resolved list because they match the pattern and are authorized via extension from their parent data stream
ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(request, metadata, authorizedIndices);
for (String dsName : expectedDataStreams) {
DataStream dataStream = metadata.dataStreams().get(dsName);
assertThat(resolvedIndices.getLocal(), not(hasItem(dsName)));
for (Index i : dataStream.getIndices()) {
assertThat(resolvedIndices.getLocal(), hasItem(i.getName()));
}
}
}
public void testBackingIndicesAreNotVisibleWhenNotIncludedByRequestWithoutWildcard() {
final User user = new User("data-stream-tester2", "data_stream_test2");
String dataStreamName = "logs-foobar";
GetAliasesRequest request = new GetAliasesRequest(dataStreamName);
assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
assertThat(request.includeDataStreams(), is(false));
// data streams and their backing indices should _not_ be in the authorized list since the backing indices
// did not match the requested pattern and the request does not support data streams
final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
assertThat(authorizedIndices, not(hasItem(dataStreamName)));
DataStream dataStream = metadata.dataStreams().get(dataStreamName);
assertThat(authorizedIndices, not(hasItem(dataStreamName)));
for (Index i : dataStream.getIndices()) {
assertThat(authorizedIndices, not(hasItem(i.getName())));
}
// neither data streams nor their backing indices will be in the resolved list since the request does not support data streams
// and the backing indices do not match the requested name
ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(request, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), not(hasItem(dataStreamName)));
for (Index i : dataStream.getIndices()) {
assertThat(resolvedIndices.getLocal(), not(hasItem(i.getName())));
}
}
public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaWildcardAndRequestThatIncludesDataStreams() {
final User user = new User("data-stream-tester2", "backing_index_test_wildcards");
String indexName = ".ds-logs-foobar-*";
SearchRequest request = new SearchRequest(indexName);
assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
assertThat(request.includeDataStreams(), is(true));
// data streams should _not_ be in the authorized list but their backing indices that matched both the requested pattern
// and the authorized pattern should be in the list
final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
assertThat(authorizedIndices, not(hasItem("logs-foobar")));
DataStream dataStream = metadata.dataStreams().get("logs-foobar");
assertThat(authorizedIndices, not(hasItem(indexName)));
for (Index i : dataStream.getIndices()) {
assertThat(authorizedIndices, hasItem(i.getName()));
}
// only the backing indices will be in the resolved list since the request does not support data streams
// but the backing indices match the requested pattern
ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(request, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), not(hasItem(dataStream.getName())));
for (Index i : dataStream.getIndices()) {
assertThat(authorizedIndices, hasItem(i.getName()));
}
}
public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaNameAndRequestThatIncludesDataStreams() {
final User user = new User("data-stream-tester2", "backing_index_test_name");
String indexName = ".ds-logs-foobar-*";
SearchRequest request = new SearchRequest(indexName);
assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
assertThat(request.includeDataStreams(), is(true));
// data streams should _not_ be in the authorized list but a single backing index that matched the requested pattern
// and the authorized name should be in the list
final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
assertThat(authorizedIndices, not(hasItem("logs-foobar")));
assertThat(authorizedIndices, contains(".ds-logs-foobar-000001"));
// only the single backing index will be in the resolved list since the request does not support data streams
// but one of the backing indices matched the requested pattern
ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(request, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), not(hasItem("logs-foobar")));
assertThat(resolvedIndices.getLocal(), contains(".ds-logs-foobar-000001"));
}
public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaWildcardAndRequestThatExcludesDataStreams() {
final User user = new User("data-stream-tester2", "backing_index_test_wildcards");
String indexName = ".ds-logs-foobar-*";
GetAliasesRequest request = new GetAliasesRequest(indexName);
assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
assertThat(request.includeDataStreams(), is(false));
// data streams should _not_ be in the authorized list but their backing indices that matched both the requested pattern
// and the authorized pattern should be in the list
final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
assertThat(authorizedIndices, not(hasItem("logs-foobar")));
DataStream dataStream = metadata.dataStreams().get("logs-foobar");
assertThat(authorizedIndices, not(hasItem(indexName)));
for (Index i : dataStream.getIndices()) {
assertThat(authorizedIndices, hasItem(i.getName()));
}
// only the backing indices will be in the resolved list since the request does not support data streams
// but the backing indices match the requested pattern
ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(request, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), not(hasItem(dataStream.getName())));
for (Index i : dataStream.getIndices()) {
assertThat(authorizedIndices, hasItem(i.getName()));
}
}
public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaNameAndRequestThatExcludesDataStreams() {
final User user = new User("data-stream-tester2", "backing_index_test_name");
String indexName = ".ds-logs-foobar-*";
GetAliasesRequest request = new GetAliasesRequest(indexName);
assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
assertThat(request.includeDataStreams(), is(false));
// data streams should _not_ be in the authorized list but a single backing index that matched the requested pattern
// and the authorized name should be in the list
final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
assertThat(authorizedIndices, not(hasItem("logs-foobar")));
assertThat(authorizedIndices, contains(".ds-logs-foobar-000001"));
// only the single backing index will be in the resolved list since the request does not support data streams
// but one of the backing indices matched the requested pattern
ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(request, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), not(hasItem("logs-foobar")));
assertThat(resolvedIndices.getLocal(), contains(".ds-logs-foobar-000001"));
}
private List<String> buildAuthorizedIndices(User user, String action) {
return buildAuthorizedIndices(user, action, TransportRequest.Empty.INSTANCE);
}
private List<String> buildAuthorizedIndices(User user, String action, TransportRequest request) {
PlainActionFuture<Role> rolesListener = new PlainActionFuture<>();
final Authentication authentication =
new Authentication(user, new RealmRef("test", "indices-aliases-resolver-tests", "node"), null);
rolesStore.getRoles(user, authentication, rolesListener);
return RBACEngine.resolveAuthorizedIndicesFromRole(rolesListener.actionGet(), action, metadata.getIndicesLookup());
return RBACEngine.resolveAuthorizedIndicesFromRole(rolesListener.actionGet(), getRequestInfo(request, action),
metadata.getIndicesLookup());
}
public static IndexMetadata.Builder indexBuilder(String index) {

43
x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java
View File

@ -11,8 +11,14 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsAction;
import org.elasticsearch.action.delete.DeleteAction;
import org.elasticsearch.action.index.IndexAction;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.DataStreamTestHelper;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.cluster.metadata.IndexAbstraction;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.collect.MapBuilder;
@ -68,12 +74,17 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
import static java.util.Collections.emptyMap;
import static org.elasticsearch.common.util.set.Sets.newHashSet;
import static org.elasticsearch.xpack.security.authz.AuthorizedIndicesTests.getRequestInfo;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.emptyIterable;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.hasItems;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.iterableWithSize;
@ -1032,6 +1043,38 @@ public class RBACEngineTests extends ESTestCase {
assertThat(response.getRunAs(), containsInAnyOrder("user01", "user02"));
}
public void testBackingIndicesAreIncludedForAuthorizedDataStreams() {
final String dataStreamName = "my_data_stream";
User user = new User(randomAlphaOfLengthBetween(4, 12));
Authentication authentication = mock(Authentication.class);
when(authentication.getUser()).thenReturn(user);
Role role = Role.builder("test1")
.cluster(Collections.singleton("all"), Collections.emptyList())
.add(IndexPrivilege.READ, dataStreamName)
.build();
TreeMap<String, IndexAbstraction> lookup = new TreeMap<>();
List<IndexMetadata> backingIndices = new ArrayList<>();
int numBackingIndices = randomIntBetween(1, 3);
for (int k = 0; k < numBackingIndices; k++) {
backingIndices.add(DataStreamTestHelper.createBackingIndex(dataStreamName, k + 1).build());
}
DataStream ds = new DataStream(dataStreamName, null,
backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()));
IndexAbstraction.DataStream iads = new IndexAbstraction.DataStream(ds, backingIndices);
lookup.put(ds.getName(), iads);
for (IndexMetadata im : backingIndices) {
lookup.put(im.getIndex().getName(), new IndexAbstraction.Index(im, iads));
}
SearchRequest request = new SearchRequest("*");
List<String> authorizedIndices =
RBACEngine.resolveAuthorizedIndicesFromRole(role, getRequestInfo(request, SearchAction.NAME), lookup);
assertThat(authorizedIndices, hasItem(dataStreamName));
assertThat(authorizedIndices, hasItems(backingIndices.stream()
.map(im -> im.getIndex().getName()).collect(Collectors.toList()).toArray(Strings.EMPTY_ARRAY)));
}
private GetUserPrivilegesResponse.Indices findIndexPrivilege(Set<GetUserPrivilegesResponse.Indices> indices, String name) {
return indices.stream().filter(i -> i.getIndices().contains(name)).findFirst().get();
}

48
x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/IndicesPermissionTests.java
View File

@ -9,6 +9,7 @@ import org.elasticsearch.ElasticsearchSecurityException;
import org.elasticsearch.Version;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.cluster.metadata.AliasMetadata;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.cluster.metadata.IndexAbstraction;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
@ -37,7 +38,9 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.stream.Collectors;
import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
@ -349,6 +352,51 @@ public class IndicesPermissionTests extends ESTestCase {
assertThat(authzMap.get(asyncSearchIndex).isGranted(), is(true));
}
public void testAuthorizationForBackingIndices() {
Metadata.Builder builder = Metadata.builder();
String dataStreamName = randomAlphaOfLength(6);
int numBackingIndices = randomIntBetween(1, 3);
List<IndexMetadata> backingIndices = new ArrayList<>();
for (int backingIndexNumber = 1; backingIndexNumber <= numBackingIndices; backingIndexNumber++) {
backingIndices.add(createIndexMetadata(DataStream.getDefaultBackingIndexName(dataStreamName, backingIndexNumber)));
}
DataStream ds = new DataStream(dataStreamName, createTimestampField("@timestamp"),
backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()));
builder.put(ds);
for (IndexMetadata index : backingIndices) {
builder.put(index, false);
}
Metadata metadata = builder.build();
FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
SortedMap<String, IndexAbstraction> lookup = metadata.getIndicesLookup();
IndicesPermission.Group group = new IndicesPermission.Group(IndexPrivilege.ALL, new FieldPermissions(), null, false,
dataStreamName);
Map<String, IndicesAccessControl.IndexAccessControl> authzMap = new IndicesPermission(group).authorize(
SearchAction.NAME,
Sets.newHashSet(backingIndices.stream().map(im -> im.getIndex().getName()).collect(Collectors.toList())),
lookup,
fieldPermissionsCache);
for (IndexMetadata im : backingIndices) {
assertThat(authzMap.get(im.getIndex().getName()).isGranted(), is(true));
}
}
private static IndexMetadata createIndexMetadata(String name) {
Settings.Builder settingsBuilder = Settings.builder()
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
.put("index.hidden", true);
IndexMetadata.Builder indexBuilder = IndexMetadata.builder(name)
.settings(settingsBuilder)
.state(IndexMetadata.State.OPEN)
.numberOfShards(1)
.numberOfReplicas(1);
return indexBuilder.build();
}
private static FieldPermissionsDefinition fieldPermissionDef(String[] granted, String[] denied) {
return new FieldPermissionsDefinition(granted, denied);
}

149
x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/50_data_streams.yml Normal file
View File

@ -0,0 +1,149 @@
---
setup:
- skip:
features: ["headers", "allowed_warnings"]
version: " - 7.99.99"
reason: "change to 7.8.99 after backport"
- do:
cluster.health:
wait_for_status: yellow
- do:
security.put_role:
name: "data_stream_role"
body: >
{
"indices": [
{ "names": ["simple*"], "privileges": ["read", "write", "view_index_metadata"] }
]
}
- do:
security.put_user:
username: "test_user"
body: >
{
"password" : "x-pack-test-password",
"roles" : [ "data_stream_role" ],
"full_name" : "user with privileges on data streams but not backing indices"
}
- do:
allowed_warnings:
- "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
indices.put_index_template:
name: my-template1
body:
index_patterns: [simple-data-stream1]
template:
mappings:
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'
---
teardown:
- do:
security.delete_user:
username: "test_user"
ignore: 404
- do:
security.delete_role:
name: "data_stream_role"
ignore: 404
---
"Test backing indices inherit parent data stream privileges":
- skip:
version: " - 7.99.99"
reason: "change to 7.8.99 after backport"
- do: # superuser
indices.create_data_stream:
name: simple-data-stream1
- is_true: acknowledged
- do: # superuser
index:
index: simple-data-stream1
id: 1
op_type: create
body: { foo: bar, "@timestamp": "2020-12-12" }
- set: { _seq_no: seqno }
- set: { _primary_term: primary_term }
- do: # superuser
indices.refresh:
index: simple-data-stream1
# should succeed since the search request is on the data stream itself
- do:
headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user
search:
rest_total_hits_as_int: true
index: simple-data-stream1
- match: { hits.total: 1 }
# should succeed since the backing index inherits the data stream's privileges
- do:
headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user
search:
rest_total_hits_as_int: true
index: .ds-simple-data-stream1-000001
- match: { hits.total: 1 }
# should succeed since the backing index inherits the data stream's privileges
- do:
headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user
index:
index: .ds-simple-data-stream1-000001
id: 1
if_seq_no: $seqno
if_primary_term: $primary_term
op_type: index
body: { foo: bar2, "@timestamp": "2020-12-12" }
- match: { _version: 2 }
- do: # superuser
indices.delete_data_stream:
name: simple-data-stream1
- is_true: acknowledged
---
"Test that requests not supporting data streams do not include data streams among authorized indices":
- skip:
version: " - 7.99.99"
reason: "change to 7.8.99 after backport"
- do: # superuser
indices.create_data_stream:
name: simple-data-stream1
- is_true: acknowledged
- do: # superuser
indices.create:
index: simple-index
body:
aliases:
simple-alias: {}
- do:
headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user
indices.get_alias:
name: simple*
- match: {simple-index.aliases.simple-alias: {}}
- is_false: simple-data-stream1
- do: # superuser
indices.delete_data_stream:
name: simple-data-stream1
- is_true: acknowledged