From 2d516d7bcc6ec291dc986b316e1ef8bef4801c63 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Mon, 6 Jul 2020 14:19:15 +0100 Subject: [PATCH] [7.x] Search all (_all, *) resolves data streams too (#58869) (#59058) Part of the original PR was merged by #59028 (cherry picked from commit 2598327726124d8a86333f79cdc45bf6a4297dbc) Signed-off-by: Andrei Dan --- .../elasticsearch/indices/DataStreamIT.java | 26 +++++++++++ .../metadata/IndexNameExpressionResolver.java | 31 +++++++------ .../IndexNameExpressionResolverTests.java | 46 +++++++++++++++++++ 3 files changed, 88 insertions(+), 15 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java index 22bd51bcaf4..814739353c1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java @@ -730,6 +730,32 @@ public class DataStreamIT extends ESIntegTestCase { assertThat(response.getIndex(), equalTo(DataStream.getDefaultBackingIndexName("logs-foobar", 1))); } + public void testSearchAllResolvesDataStreams() throws Exception { + putComposableIndexTemplate("id1", "@timestamp1", List.of("metrics-foo*")); + CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-foo"); + client().admin().indices().createDataStream(createDataStreamRequest).get(); + + putComposableIndexTemplate("id2", "@timestamp2", List.of("metrics-bar*")); + createDataStreamRequest = new CreateDataStreamAction.Request("metrics-bar"); + client().admin().indices().createDataStream(createDataStreamRequest).get(); + + int numDocsBar = randomIntBetween(2, 16); + indexDocs("metrics-bar", "@timestamp2", numDocsBar); + int numDocsFoo = randomIntBetween(2, 16); + indexDocs("metrics-foo", "@timestamp1", numDocsFoo); + + RolloverResponse rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest("metrics-foo", null)).get(); + assertThat(rolloverResponse.getNewIndex(), equalTo(DataStream.getDefaultBackingIndexName("metrics-foo", 2))); + + // ingest some more data in the rolled data stream + int numDocsRolledFoo = randomIntBetween(2, 16); + indexDocs("metrics-foo", "@timestamp1", numDocsRolledFoo); + + SearchRequest searchRequest = new SearchRequest("*"); + SearchResponse searchResponse = client().search(searchRequest).actionGet(); + assertThat(searchResponse.getHits().getTotalHits().value, is((long) numDocsBar + numDocsFoo + numDocsRolledFoo)); + } + private static void assertBackingIndex(String backingIndex, String timestampFieldPathInMapping) { assertBackingIndex(backingIndex, timestampFieldPathInMapping, Map.of("type", "date")); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 2d2038c7cdd..96130cf931a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -176,7 +176,6 @@ public class IndexNameExpressionResolver { } Metadata metadata = context.getState().metadata(); IndicesOptions options = context.getOptions(); - final boolean failClosed = options.forbidClosedIndices() && options.ignoreUnavailable() == false; // If only one index is specified then whether we fail a request if an index is missing depends on the allow_no_indices // option. At some point we should change this, because there shouldn't be a reason why whether a single index // or multiple indices are specified yield different behaviour. @@ -260,20 +259,8 @@ public class IndexNameExpressionResolver { } for (IndexMetadata index : indexAbstraction.getIndices()) { - if (index.getState() == IndexMetadata.State.CLOSE) { - if (failClosed) { - throw new IndexClosedException(index.getIndex()); - } else { - if (options.forbidClosedIndices() == false && addIndex(index, context)) { - concreteIndices.add(index.getIndex()); - } - } - } else if (index.getState() == IndexMetadata.State.OPEN) { - if (addIndex(index, context)) { - concreteIndices.add(index.getIndex()); - } - } else { - throw new IllegalStateException("index state [" + index.getState() + "] not supported"); + if (shouldTrackConcreteIndex(context, options, index)) { + concreteIndices.add(index.getIndex()); } } } @@ -287,6 +274,20 @@ public class IndexNameExpressionResolver { return concreteIndices.toArray(new Index[concreteIndices.size()]); } + private static boolean shouldTrackConcreteIndex(Context context, IndicesOptions options, IndexMetadata index) { + if (index.getState() == IndexMetadata.State.CLOSE) { + if (options.forbidClosedIndices() && options.ignoreUnavailable() == false) { + throw new IndexClosedException(index.getIndex()); + } else { + return options.forbidClosedIndices() == false && addIndex(index, context); + } + } else if (index.getState() == IndexMetadata.State.OPEN) { + return addIndex(index, context); + } else { + throw new IllegalStateException("index state [" + index.getState() + "] not supported"); + } + } + private static boolean addIndex(IndexMetadata metadata, Context context) { // This used to check the `index.search.throttled` setting, but we eventually decided that it was // trappy to hide throttled indices by default. In order to avoid breaking backward compatibility, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index 57e30db5455..cef0f835cc3 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -1910,6 +1910,17 @@ public class IndexNameExpressionResolverTests extends ESTestCase { assertThat(result[2].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 1))); assertThat(result[3].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 2)));; } + { + IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN; + Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, + randomFrom(new String[]{"*"}, new String[]{"_all"}, new String[0])); + Arrays.sort(result, Comparator.comparing(Index::getName)); + assertThat(result.length, equalTo(4)); + assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 1))); + assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 2))); + assertThat(result[2].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 1))); + assertThat(result[3].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 2)));; + } { IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN; Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "logs-m*"); @@ -1925,6 +1936,41 @@ public class IndexNameExpressionResolverTests extends ESTestCase { } } + public void testDataStreamsWithClosedBackingIndicesAndWildcardExpressions() { + final String dataStream1 = "logs-mysql"; + final String dataStream2 = "logs-redis"; + IndexMetadata index1 = createBackingIndex(dataStream1, 1).state(State.CLOSE).build(); + IndexMetadata index2 = createBackingIndex(dataStream1, 2).build(); + IndexMetadata index3 = createBackingIndex(dataStream2, 1).state(State.CLOSE).build(); + IndexMetadata index4 = createBackingIndex(dataStream2, 2).build(); + Metadata.Builder mdBuilder = Metadata.builder() + .put(index1, false) + .put(index2, false) + .put(index3, false) + .put(index4, false) + .put(new DataStream(dataStream1, createTimestampField("@timestamp"), + org.elasticsearch.common.collect.List.of(index1.getIndex(), index2.getIndex()))) + .put(new DataStream(dataStream2, createTimestampField("@timestamp"), + org.elasticsearch.common.collect.List.of(index3.getIndex(), index4.getIndex()))); + + ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); + IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN; + { + Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "logs-*"); + Arrays.sort(result, Comparator.comparing(Index::getName)); + assertThat(result.length, equalTo(2)); + assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 2))); + assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 2)));; + } + { + Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "*"); + Arrays.sort(result, Comparator.comparing(Index::getName)); + assertThat(result.length, equalTo(2)); + assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 2))); + assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 2)));; + } + } + public void testDataStreamsWithRegularIndexAndAlias() { final String dataStream1 = "logs-foobar"; IndexMetadata index1 = createBackingIndex(dataStream1, 1).build();