[7.x] Ignore matching data streams if include_data_streams is false (#59028)

This commit is contained in:
Dan Hermann 2020-07-03 14:51:32 -05:00 committed by GitHub
parent b9d9964d10
commit 7c43cbca82
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 138 additions and 86 deletions

View File

@ -41,7 +41,7 @@
- match: { hits.hits.0._source.foo: 'bar' }
- do:
catch: bad_request
catch: missing
indices.delete:
index: logs-foobar

View File

@ -35,7 +35,6 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
import org.elasticsearch.action.admin.indices.template.delete.DeleteComposableIndexTemplateAction;
import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction;
import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder;
import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
@ -208,32 +207,21 @@ public class DataStreamIT extends ESIntegTestCase {
client().admin().indices().createDataStream(createDataStreamRequest).get();
{
BulkRequest bulkRequest = new BulkRequest()
.add(new IndexRequest(dataStreamName).source("{}", XContentType.JSON));
expectFailure(dataStreamName, () -> client().bulk(bulkRequest).actionGet());
}
{
BulkRequest bulkRequest = new BulkRequest()
.add(new DeleteRequest(dataStreamName, "_id"));
expectFailure(dataStreamName, () -> client().bulk(bulkRequest).actionGet());
}
{
BulkRequest bulkRequest = new BulkRequest()
.add(new UpdateRequest(dataStreamName, "_id").doc("{}", XContentType.JSON));
expectFailure(dataStreamName, () -> client().bulk(bulkRequest).actionGet());
}
{
IndexRequest indexRequest = new IndexRequest(dataStreamName).source("{}", XContentType.JSON);
expectFailure(dataStreamName, () -> client().index(indexRequest).actionGet());
IndexRequest indexRequest = new IndexRequest(dataStreamName)
.source("{\"@timestamp1\": \"2020-12-12\"}", XContentType.JSON);
Exception e = expectThrows(IndexNotFoundException.class, () -> client().index(indexRequest).actionGet());
assertThat(e.getMessage(), equalTo("no such index [null]"));
}
{
UpdateRequest updateRequest = new UpdateRequest(dataStreamName, "_id")
.doc("{}", XContentType.JSON);
expectFailure(dataStreamName, () -> client().update(updateRequest).actionGet());
Exception e = expectThrows(IndexNotFoundException.class, () -> client().update(updateRequest).actionGet());
assertThat(e.getMessage(), equalTo("no such index [null]"));
}
{
DeleteRequest deleteRequest = new DeleteRequest(dataStreamName, "_id");
expectFailure(dataStreamName, () -> client().delete(deleteRequest).actionGet());
Exception e = expectThrows(IndexNotFoundException.class, () -> client().delete(deleteRequest).actionGet());
assertThat(e.getMessage(), equalTo("no such index [null]"));
}
{
IndexRequest indexRequest = new IndexRequest(dataStreamName).source("{}", XContentType.JSON)
@ -424,7 +412,7 @@ public class DataStreamIT extends ESIntegTestCase {
verifyResolvability(wildcardExpression, client().admin().indices().prepareUpgrade(wildcardExpression), false);
verifyResolvability(wildcardExpression, client().admin().indices().prepareRecoveries(wildcardExpression), false);
verifyResolvability(wildcardExpression, client().admin().indices().prepareUpgradeStatus(wildcardExpression), false);
verifyResolvability(wildcardExpression, getAliases(wildcardExpression), true);
verifyResolvability(wildcardExpression, getAliases(wildcardExpression), false);
verifyResolvability(wildcardExpression, getFieldMapping(wildcardExpression), false);
verifyResolvability(wildcardExpression,
putMapping("{\"_doc\":{\"properties\": {\"my_field\":{\"type\":\"keyword\"}}}}", wildcardExpression), false);
@ -473,7 +461,8 @@ public class DataStreamIT extends ESIntegTestCase {
.index(dataStreamName).aliases("foo");
IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest();
aliasesAddRequest.addAliasAction(addAction);
expectFailure(dataStreamName, () -> client().admin().indices().aliases(aliasesAddRequest).actionGet());
Exception e = expectThrows(IndexNotFoundException.class, () -> client().admin().indices().aliases(aliasesAddRequest).actionGet());
assertThat(e.getMessage(), equalTo("no such index [" + dataStreamName +"]"));
}
public void testAliasActionsFailOnDataStreamBackingIndices() throws Exception {
@ -729,8 +718,7 @@ public class DataStreamIT extends ESIntegTestCase {
private static void verifyResolvability(String dataStream, ActionRequestBuilder requestBuilder, boolean fail, long expectedCount) {
if (fail) {
String expectedErrorMessage = "The provided expression [" + dataStream +
"] matches a data stream, specify the corresponding concrete indices instead.";
String expectedErrorMessage = "no such index [" + dataStream + "]";
if (requestBuilder instanceof MultiSearchRequestBuilder) {
MultiSearchResponse multiSearchResponse = ((MultiSearchRequestBuilder) requestBuilder).get();
assertThat(multiSearchResponse.getResponses().length, equalTo(1));
@ -738,10 +726,10 @@ public class DataStreamIT extends ESIntegTestCase {
assertThat(multiSearchResponse.getResponses()[0].getFailure(), instanceOf(IllegalArgumentException.class));
assertThat(multiSearchResponse.getResponses()[0].getFailure().getMessage(), equalTo(expectedErrorMessage));
} else if (requestBuilder instanceof ValidateQueryRequestBuilder) {
ValidateQueryResponse response = (ValidateQueryResponse) requestBuilder.get();
assertThat(response.getQueryExplanation().get(0).getError(), equalTo(expectedErrorMessage));
Exception e = expectThrows(IndexNotFoundException.class, requestBuilder::get);
assertThat(e.getMessage(), equalTo(expectedErrorMessage));
} else {
Exception e = expectThrows(IllegalArgumentException.class, requestBuilder::get);
Exception e = expectThrows(IndexNotFoundException.class, requestBuilder::get);
assertThat(e.getMessage(), equalTo(expectedErrorMessage));
}
} else {

View File

@ -229,7 +229,7 @@ public class IndexNameExpressionResolver {
}
} else if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM &&
context.includeDataStreams() == false) {
throw dataStreamsNotSupportedException(expression);
continue;
}
if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS && context.isResolveToWriteIndex()) {
@ -301,11 +301,6 @@ public class IndexNameExpressionResolver {
"alias, specify the corresponding concrete indices instead.");
}
private static IllegalArgumentException dataStreamsNotSupportedException(String expression) {
return new IllegalArgumentException("The provided expression [" + expression + "] matches a " +
"data stream, specify the corresponding concrete indices instead.");
}
/**
* Utility method that allows to resolve an index expression to its corresponding single concrete index.
* Callers should make sure they provide proper {@link org.elasticsearch.action.support.IndicesOptions}
@ -357,7 +352,12 @@ public class IndexNameExpressionResolver {
*/
public Index concreteWriteIndex(ClusterState state, IndicesOptions options, String index, boolean allowNoIndices,
boolean includeDataStreams) {
Context context = new Context(state, options, false, true, includeDataStreams);
IndicesOptions combinedOptions = IndicesOptions.fromOptions(options.ignoreUnavailable(), allowNoIndices,
options.expandWildcardsOpen(), options.expandWildcardsClosed(), options.expandWildcardsHidden(),
options.allowAliasesToMultipleIndices(), options.forbidClosedIndices(), options.ignoreAliases(),
options.ignoreThrottled());
Context context = new Context(state, combinedOptions, false, true, includeDataStreams);
Index[] indices = concreteIndices(context, index);
if (allowNoIndices && indices.length == 0) {
return null;
@ -732,10 +732,20 @@ public class IndexNameExpressionResolver {
}
if (isEmptyOrTrivialWildcard(expressions)) {
if (context.includeDataStreams() == false && metadata.dataStreams().isEmpty() == false) {
throw dataStreamsNotSupportedException(expressions.toString());
List<String> resolvedExpressions = resolveEmptyOrTrivialWildcard(options, metadata);
if (context.includeDataStreams()) {
final IndexMetadata.State excludeState = excludeState(options);
final Map<String, IndexAbstraction> dataStreamsAbstractions = metadata.getIndicesLookup().entrySet()
.stream()
.filter(entry -> entry.getValue().getType() == IndexAbstraction.Type.DATA_STREAM)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
// dedup backing indices if expand hidden indices option is true
Set<String> resolvedIncludingDataStreams = new HashSet<>(resolvedExpressions);
resolvedIncludingDataStreams.addAll(expand(context, excludeState, dataStreamsAbstractions,
expressions.isEmpty() ? "_all" : expressions.get(0), options.expandWildcardsHidden()));
return new ArrayList<>(resolvedIncludingDataStreams);
}
return resolveEmptyOrTrivialWildcard(options, metadata);
return resolvedExpressions;
}
Set<String> result = innerResolve(context, expressions, options, metadata);
@ -786,8 +796,8 @@ public class IndexNameExpressionResolver {
} else if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS && options.ignoreAliases()) {
throw aliasesNotSupportedException(expression);
} else if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM &&
context.includeDataStreams() == false) {
throw dataStreamsNotSupportedException(expression);
context.includeDataStreams() == false) {
throw indexNotFoundException(expression);
}
}
if (add) {
@ -867,7 +877,7 @@ public class IndexNameExpressionResolver {
public static Map<String, IndexAbstraction> matches(Context context, Metadata metadata, String expression) {
if (Regex.isMatchAllPattern(expression)) {
return filterIndicesLookup(context, metadata.getIndicesLookup(), null, expression, context.getOptions());
return filterIndicesLookup(context, metadata.getIndicesLookup(), null, context.getOptions());
} else if (expression.indexOf("*") == expression.length() - 1) {
return suffixWildcard(context, metadata, expression);
} else {
@ -882,18 +892,17 @@ public class IndexNameExpressionResolver {
toPrefixCharArr[toPrefixCharArr.length - 1]++;
String toPrefix = new String(toPrefixCharArr);
SortedMap<String, IndexAbstraction> subMap = metadata.getIndicesLookup().subMap(fromPrefix, toPrefix);
return filterIndicesLookup(context, subMap, null, expression, context.getOptions());
return filterIndicesLookup(context, subMap, null, context.getOptions());
}
private static Map<String, IndexAbstraction> otherWildcard(Context context, Metadata metadata, String expression) {
final String pattern = expression;
return filterIndicesLookup(context, metadata.getIndicesLookup(), e -> Regex.simpleMatch(pattern, e.getKey()),
expression, context.getOptions());
context.getOptions());
}
private static Map<String, IndexAbstraction> filterIndicesLookup(Context context, SortedMap<String, IndexAbstraction> indicesLookup,
Predicate<? super Map.Entry<String, IndexAbstraction>> filter,
String expression,
IndicesOptions options) {
boolean shouldConsumeStream = false;
Stream<Map.Entry<String, IndexAbstraction>> stream = indicesLookup.entrySet().stream();
@ -907,11 +916,7 @@ public class IndexNameExpressionResolver {
}
if (context.includeDataStreams() == false) {
shouldConsumeStream = true;
stream = stream.peek(e -> {
if (e.getValue().getType() == IndexAbstraction.Type.DATA_STREAM) {
throw dataStreamsNotSupportedException(expression);
}
});
stream = stream.filter(e -> e.getValue().getType() != IndexAbstraction.Type.DATA_STREAM);
}
if (shouldConsumeStream) {
return stream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

View File

@ -1716,27 +1716,25 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
{
IndicesAliasesRequest.AliasActions aliasActions = IndicesAliasesRequest.AliasActions.add().index(dataStreamName);
IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
IndexNotFoundException iae = expectThrows(IndexNotFoundException.class,
() -> indexNameExpressionResolver.concreteIndexNames(state, aliasActions));
assertEquals("The provided expression [" + dataStreamName + "] matches a data stream, specify the corresponding " +
"concrete indices instead.", iae.getMessage());
assertEquals("no such index [" + dataStreamName + "]", iae.getMessage());
}
{
IndicesAliasesRequest.AliasActions aliasActions = IndicesAliasesRequest.AliasActions.add().index("my-data-*").alias("my-data");
IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
IndexNotFoundException iae = expectThrows(IndexNotFoundException.class,
() -> indexNameExpressionResolver.concreteIndexNames(state, aliasActions));
assertEquals("The provided expression [my-data-*] matches a data stream, specify the corresponding concrete indices instead.",
iae.getMessage());
assertEquals("no such index [my-data-*]", iae.getMessage());
}
//{
// IndicesAliasesRequest.AliasActions aliasActions = IndicesAliasesRequest.AliasActions.add().index(dataStreamName)
// .alias("my-data");
// String[] indices = indexNameExpressionResolver.concreteIndexNames(state, aliasActions);
// assertEquals(1, indices.length);
// assertEquals(backingIndex.getIndex().getName(), indices[0]);
//}
{
IndicesAliasesRequest.AliasActions aliasActions = IndicesAliasesRequest.AliasActions.add().index(dataStreamName)
.alias("my-data");
IndexNotFoundException iae = expectThrows(IndexNotFoundException.class,
() -> indexNameExpressionResolver.concreteIndexNames(state, aliasActions));
assertEquals("no such index [" + dataStreamName + "]", iae.getMessage());
}
}
public void testInvalidIndex() {
@ -1835,28 +1833,24 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
{
// Ignore data streams
IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
Exception e = expectThrows(IllegalArgumentException.class,
Exception e = expectThrows(IndexNotFoundException.class,
() -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, false, "my-data-stream"));
assertThat(e.getMessage(), equalTo("The provided expression [my-data-stream] matches a " +
"data stream, specify the corresponding concrete indices instead."));
assertThat(e.getMessage(), equalTo("no such index [my-data-stream]"));
}
{
// Ignore data streams and allow no indices
IndicesOptions indicesOptions = new IndicesOptions(EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES),
EnumSet.of(IndicesOptions.WildcardStates.OPEN));
Exception e = expectThrows(IllegalArgumentException.class,
Exception e = expectThrows(IndexNotFoundException.class,
() -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, false, "my-data-stream"));
assertThat(e.getMessage(), equalTo("The provided expression [my-data-stream] matches a " +
"data stream, specify the corresponding concrete indices instead."));
assertThat(e.getMessage(), equalTo("no such index [my-data-stream]"));
}
{
// Ignore data streams, allow no indices and ignore unavailable
IndicesOptions indicesOptions = new IndicesOptions(EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES,
IndicesOptions.Option.IGNORE_UNAVAILABLE), EnumSet.of(IndicesOptions.WildcardStates.OPEN));
Exception e = expectThrows(IllegalArgumentException.class,
() -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, false, "my-data-stream"));
assertThat(e.getMessage(), equalTo("The provided expression [my-data-stream] matches a " +
"data stream, specify the corresponding concrete indices instead."));
Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, false, "my-data-stream");
assertThat(result.length, equalTo(0));
}
{
IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
@ -1867,28 +1861,24 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
// Ignore data streams
IndicesOptions indicesOptions = new IndicesOptions(EnumSet.noneOf(IndicesOptions.Option.class),
EnumSet.of(IndicesOptions.WildcardStates.OPEN));
Exception e = expectThrows(IllegalArgumentException.class,
Exception e = expectThrows(IndexNotFoundException.class,
() -> indexNameExpressionResolver.concreteWriteIndex(state, indicesOptions, "my-data-stream", true, false));
assertThat(e.getMessage(), equalTo("The provided expression [my-data-stream] matches a " +
"data stream, specify the corresponding concrete indices instead."));
assertThat(e.getMessage(), equalTo("no such index [my-data-stream]"));
}
{
// Ignore data streams and allow no indices
IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
Exception e = expectThrows(IllegalArgumentException.class,
Exception e = expectThrows(IndexNotFoundException.class,
() -> indexNameExpressionResolver.concreteWriteIndex(state, indicesOptions, "my-data-stream", false, false));
assertThat(e.getMessage(), equalTo("The provided expression [my-data-stream] matches a data stream, " +
"specify the corresponding concrete indices instead."));
assertThat(e.getMessage(), equalTo("no such index [my-data-stream]"));
}
{
// Ignore data streams, allow no indices and ignore unavailable
IndicesOptions indicesOptions = new IndicesOptions(EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES,
IndicesOptions.Option.IGNORE_UNAVAILABLE), EnumSet.of(IndicesOptions.WildcardStates.OPEN));
Exception e = expectThrows(IllegalArgumentException.class,
Exception e = expectThrows(IndexNotFoundException.class,
() -> indexNameExpressionResolver.concreteWriteIndex(state, indicesOptions, "my-data-stream", false, false));
assertThat(e.getMessage(), equalTo("The provided expression [my-data-stream] matches a data stream, " +
"specify the corresponding concrete indices instead."));
assertThat(e.getMessage(), equalTo("no such index [null]"));
}
}
@ -1930,10 +1920,8 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
}
{
IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN; // without include data streams
Exception e = expectThrows(IllegalArgumentException.class,
() -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, "logs-*"));
assertThat(e.getMessage(), equalTo("The provided expression [logs-*] matches a data stream, " +
"specify the corresponding concrete indices instead."));
Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, "logs-*");
assertThat(result.length, equalTo(0));
}
}

View File

@ -32,6 +32,8 @@ import java.util.Collections;
import java.util.List;
import java.util.Set;
import static org.elasticsearch.cluster.DataStreamTestHelper.createBackingIndex;
import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField;
import static org.elasticsearch.common.util.set.Sets.newHashSet;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
@ -207,6 +209,75 @@ public class WildcardExpressionResolverTests extends ESTestCase {
}
}
public void testResolveDataStreams() {
String dataStreamName = "foo_logs";
IndexMetadata firstBackingIndexMetadata = createBackingIndex(dataStreamName, 1).build();
IndexMetadata secondBackingIndexMetadata = createBackingIndex(dataStreamName, 2).build();
Metadata.Builder mdBuilder = Metadata.builder()
.put(indexBuilder("foo_foo").state(State.OPEN))
.put(indexBuilder("bar_bar").state(State.OPEN))
.put(indexBuilder("foo_index").state(State.OPEN).putAlias(AliasMetadata.builder("foo_alias")))
.put(indexBuilder("bar_index").state(State.OPEN).putAlias(AliasMetadata.builder("foo_alias")))
.put(firstBackingIndexMetadata, true)
.put(secondBackingIndexMetadata, true)
.put(new DataStream(dataStreamName, createTimestampField("timestamp"),
org.elasticsearch.common.collect.List.of(firstBackingIndexMetadata.getIndex(), secondBackingIndexMetadata.getIndex())));
ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
IndexNameExpressionResolver.WildcardExpressionResolver resolver = new IndexNameExpressionResolver.WildcardExpressionResolver();
{
IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false, true, false,
false, false);
IndexNameExpressionResolver.Context indicesAndAliasesContext =
new IndexNameExpressionResolver.Context(state, indicesAndAliasesOptions);
// data streams are not included but expression matches the data stream
List<String> indices = resolver.resolve(indicesAndAliasesContext, Collections.singletonList("foo_*"));
assertThat(indices, containsInAnyOrder("foo_index", "foo_foo", "bar_index"));
// data streams are not included and expression doesn't match the data steram
indices = resolver.resolve(indicesAndAliasesContext, Collections.singletonList("bar_*"));
assertThat(indices, containsInAnyOrder("bar_bar", "bar_index"));
}
{
IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false, true, false,
false, false);
IndexNameExpressionResolver.Context indicesAliasesAndDataStreamsContext = new IndexNameExpressionResolver.Context(state,
indicesAndAliasesOptions, false, false, true);
// data stream's corresponding backing indices are resolved
List<String> indices = resolver.resolve(indicesAliasesAndDataStreamsContext, Collections.singletonList("foo_*"));
assertThat(indices, containsInAnyOrder("foo_index", "bar_index", "foo_foo", ".ds-foo_logs-000001",
".ds-foo_logs-000002"));
// include all wildcard adds the data stream's backing indices
indices = resolver.resolve(indicesAliasesAndDataStreamsContext, Collections.singletonList("*"));
assertThat(indices, containsInAnyOrder("foo_index", "bar_index", "foo_foo", "bar_bar", ".ds-foo_logs-000001",
".ds-foo_logs-000002"));
}
{
IndicesOptions indicesAliasesAndExpandHiddenOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false,
true, true, false, false, false);
IndexNameExpressionResolver.Context indicesAliasesDataStreamsAndHiddenIndices = new IndexNameExpressionResolver.Context(state,
indicesAliasesAndExpandHiddenOptions, false, false, true);
// data stream's corresponding backing indices are resolved
List<String> indices = resolver.resolve(indicesAliasesDataStreamsAndHiddenIndices, Collections.singletonList("foo_*"));
assertThat(indices, containsInAnyOrder("foo_index", "bar_index", "foo_foo", ".ds-foo_logs-000001",
".ds-foo_logs-000002"));
// include all wildcard adds the data stream's backing indices
indices = resolver.resolve(indicesAliasesDataStreamsAndHiddenIndices, Collections.singletonList("*"));
assertThat(indices, containsInAnyOrder("foo_index", "bar_index", "foo_foo", "bar_bar", ".ds-foo_logs-000001",
".ds-foo_logs-000002"));
}
}
public void testMatchesConcreteIndicesWildcardAndAliases() {
Metadata.Builder mdBuilder = Metadata.builder()
.put(indexBuilder("foo_foo").state(State.OPEN))