honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[7.x] Search all (_all, *) resolves data streams too (#58869) (#59058) 

Part of the original PR was merged by #59028

(cherry picked from commit 2598327726124d8a86333f79cdc45bf6a4297dbc)
Signed-off-by: Andrei Dan <andrei.dan@elastic.co>
This commit is contained in:
Andrei Dan 2020-07-06 14:19:15 +01:00 committed by GitHub
parent 550dcb0ca6
commit 2d516d7bcc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 88 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java
View File

@ -730,6 +730,32 @@ public class DataStreamIT extends ESIntegTestCase {
assertThat(response.getIndex(), equalTo(DataStream.getDefaultBackingIndexName("logs-foobar", 1)));
}
public void testSearchAllResolvesDataStreams() throws Exception {
putComposableIndexTemplate("id1", "@timestamp1", List.of("metrics-foo*"));
CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-foo");
client().admin().indices().createDataStream(createDataStreamRequest).get();
putComposableIndexTemplate("id2", "@timestamp2", List.of("metrics-bar*"));
createDataStreamRequest = new CreateDataStreamAction.Request("metrics-bar");
client().admin().indices().createDataStream(createDataStreamRequest).get();
int numDocsBar = randomIntBetween(2, 16);
indexDocs("metrics-bar", "@timestamp2", numDocsBar);
int numDocsFoo = randomIntBetween(2, 16);
indexDocs("metrics-foo", "@timestamp1", numDocsFoo);
RolloverResponse rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest("metrics-foo", null)).get();
assertThat(rolloverResponse.getNewIndex(), equalTo(DataStream.getDefaultBackingIndexName("metrics-foo", 2)));
// ingest some more data in the rolled data stream
int numDocsRolledFoo = randomIntBetween(2, 16);
indexDocs("metrics-foo", "@timestamp1", numDocsRolledFoo);
SearchRequest searchRequest = new SearchRequest("*");
SearchResponse searchResponse = client().search(searchRequest).actionGet();
assertThat(searchResponse.getHits().getTotalHits().value, is((long) numDocsBar + numDocsFoo + numDocsRolledFoo));
}
private static void assertBackingIndex(String backingIndex, String timestampFieldPathInMapping) {
assertBackingIndex(backingIndex, timestampFieldPathInMapping, Map.of("type", "date"));
}

31
server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
View File

@ -176,7 +176,6 @@ public class IndexNameExpressionResolver {
}
Metadata metadata = context.getState().metadata();
IndicesOptions options = context.getOptions();
final boolean failClosed = options.forbidClosedIndices() && options.ignoreUnavailable() == false;
// If only one index is specified then whether we fail a request if an index is missing depends on the allow_no_indices
// option. At some point we should change this, because there shouldn't be a reason why whether a single index
// or multiple indices are specified yield different behaviour.
@ -260,20 +259,8 @@ public class IndexNameExpressionResolver {
}
for (IndexMetadata index : indexAbstraction.getIndices()) {
if (index.getState() == IndexMetadata.State.CLOSE) {
if (failClosed) {
throw new IndexClosedException(index.getIndex());
} else {
if (options.forbidClosedIndices() == false && addIndex(index, context)) {
concreteIndices.add(index.getIndex());
}
}
} else if (index.getState() == IndexMetadata.State.OPEN) {
if (addIndex(index, context)) {
concreteIndices.add(index.getIndex());
}
} else {
throw new IllegalStateException("index state [" + index.getState() + "] not supported");
if (shouldTrackConcreteIndex(context, options, index)) {
concreteIndices.add(index.getIndex());
}
}
}
@ -287,6 +274,20 @@ public class IndexNameExpressionResolver {
return concreteIndices.toArray(new Index[concreteIndices.size()]);
}
private static boolean shouldTrackConcreteIndex(Context context, IndicesOptions options, IndexMetadata index) {
if (index.getState() == IndexMetadata.State.CLOSE) {
if (options.forbidClosedIndices() && options.ignoreUnavailable() == false) {
throw new IndexClosedException(index.getIndex());
} else {
return options.forbidClosedIndices() == false && addIndex(index, context);
}
} else if (index.getState() == IndexMetadata.State.OPEN) {
return addIndex(index, context);
} else {
throw new IllegalStateException("index state [" + index.getState() + "] not supported");
}
}
private static boolean addIndex(IndexMetadata metadata, Context context) {
// This used to check the `index.search.throttled` setting, but we eventually decided that it was
// trappy to hide throttled indices by default. In order to avoid breaking backward compatibility,

46
server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
View File

@ -1910,6 +1910,17 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
assertThat(result[2].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 1)));
assertThat(result[3].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 2)));;
}
{
IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true,
randomFrom(new String[]{"*"}, new String[]{"_all"}, new String[0]));
Arrays.sort(result, Comparator.comparing(Index::getName));
assertThat(result.length, equalTo(4));
assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 1)));
assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 2)));
assertThat(result[2].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 1)));
assertThat(result[3].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 2)));;
}
{
IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "logs-m*");
@ -1925,6 +1936,41 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
}
}
public void testDataStreamsWithClosedBackingIndicesAndWildcardExpressions() {
final String dataStream1 = "logs-mysql";
final String dataStream2 = "logs-redis";
IndexMetadata index1 = createBackingIndex(dataStream1, 1).state(State.CLOSE).build();
IndexMetadata index2 = createBackingIndex(dataStream1, 2).build();
IndexMetadata index3 = createBackingIndex(dataStream2, 1).state(State.CLOSE).build();
IndexMetadata index4 = createBackingIndex(dataStream2, 2).build();
Metadata.Builder mdBuilder = Metadata.builder()
.put(index1, false)
.put(index2, false)
.put(index3, false)
.put(index4, false)
.put(new DataStream(dataStream1, createTimestampField("@timestamp"),
org.elasticsearch.common.collect.List.of(index1.getIndex(), index2.getIndex())))
.put(new DataStream(dataStream2, createTimestampField("@timestamp"),
org.elasticsearch.common.collect.List.of(index3.getIndex(), index4.getIndex())));
ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
{
Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "logs-*");
Arrays.sort(result, Comparator.comparing(Index::getName));
assertThat(result.length, equalTo(2));
assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 2)));
assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 2)));;
}
{
Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "*");
Arrays.sort(result, Comparator.comparing(Index::getName));
assertThat(result.length, equalTo(2));
assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 2)));
assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 2)));;
}
}
public void testDataStreamsWithRegularIndexAndAlias() {
final String dataStream1 = "logs-foobar";
IndexMetadata index1 = createBackingIndex(dataStream1, 1).build();