honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make data streams in APIs resolvable. (#55337) 

Backport from: #54726

The INCLUDE_DATA_STREAMS indices option controls whether data streams can be resolved in an api for both concrete names and wildcard expressions. If data streams cannot be resolved then a 400 error is returned indicating that data streams cannot be used.

In this pr, the INCLUDE_DATA_STREAMS indices option is enabled in the following APIs: search, msearch, refresh, index (op_type create only) and bulk (index requests with op type create only). In a subsequent later change, we will determine which other APIs need to be able to resolve data streams and enable the INCLUDE_DATA_STREAMS indices option for these APIs.

Whether an api resolve all backing indices of a data stream or the latest index of a data stream (write index) depends on the IndexNameExpressionResolver.Context.isResolveToWriteIndex().
If isResolveToWriteIndex() returns true then data streams resolve to the latest index (for example: index api) and otherwise a data stream resolves to all backing indices of a data stream (for example: search api).

Relates to #53100
This commit is contained in:
Martijn van Groningen 2020-04-17 08:33:37 +02:00 committed by GitHub
parent bed422162c
commit 417d5f2009
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
36 changed files with 1064 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
View File

@ -1349,8 +1349,9 @@ public class RequestConvertersTests extends ESTestCase {
IndicesOptions msearchDefault = new MultiSearchRequest().indicesOptions();
searchRequest.indicesOptions(IndicesOptions.fromOptions(randomlyGenerated.ignoreUnavailable(),
randomlyGenerated.allowNoIndices(), randomlyGenerated.expandWildcardsOpen(), randomlyGenerated.expandWildcardsClosed(),
msearchDefault.allowAliasesToMultipleIndices(), msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases(),
msearchDefault.ignoreThrottled()));
msearchDefault.expandWildcardsHidden(), msearchDefault.allowAliasesToMultipleIndices(),
msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases(), msearchDefault.ignoreThrottled(),
msearchDefault.includeDataStreams()));
multiSearchRequest.add(searchRequest);
}

2
modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java
View File

@ -45,7 +45,7 @@ public class MultiSearchTemplateRequest extends ActionRequest implements Composi
private int maxConcurrentSearchRequests = 0;
private List<SearchTemplateRequest> requests = new ArrayList<>();
private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled();
private IndicesOptions indicesOptions = IndicesOptions.strictIncludeDataStreamsExpandOpenAndForbidClosedIgnoreThrottled();
public MultiSearchTemplateRequest() {}

33
rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
View File

@ -1,8 +1,8 @@
---
"Create data stream":
- skip:
version: " - 7.99.99"
reason: "enable in 7.8+ after back-porting https://github.com/elastic/elasticsearch/pull/54467"
version: " - 7.7.99"
reason: available only in 7.8+
- do:
indices.create_data_stream:
@ -30,6 +30,23 @@
- length: { 1.indices: 1 }
- match: { 1.indices.0.index_name: 'simple-data-stream2-000001' }
- do:
index:
index: simple-data-stream1
body: { foo: bar }
- do:
indices.refresh:
index: simple-data-stream1
- do:
search:
index: simple-data-stream1
body: { query: { match_all: {} } }
- length: { hits.hits: 1 }
- match: { hits.hits.0._index: simple-data-stream1-000001 }
- match: { hits.hits.0._source.foo: 'bar' }
- do:
indices.delete_data_stream:
name: simple-data-stream1
@ -59,8 +76,8 @@
---
"Get data stream":
- skip:
version: " - 7.99.99"
reason: available only in 7.7+
version: " - 7.7.99"
reason: available only in 7.8+
- do:
indices.create_data_stream:
@ -123,8 +140,8 @@
---
"Delete data stream with backing indices":
- skip:
version: " - 7.99.99"
reason: "enable in 7.8+ after back-porting"
version: " - 7.7.99"
reason: available only in 7.8+
- do:
indices.create_data_stream:
@ -143,8 +160,7 @@
- do:
indices.get:
index: "*"
expand_wildcards: all
index: ['delete-data-stream1-000001', 'test_index']
- is_true: test_index.settings
- is_true: delete-data-stream1-000001.settings
@ -165,4 +181,3 @@
catch: missing
indices.get:
index: "delete-data-stream1-000001"
expand_wildcards: all

67
rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml Normal file
View File

@ -0,0 +1,67 @@
---
"Test apis that do not supported data streams":
- skip:
version: " - 7.7.99"
reason: available only in 7.8+
- do:
indices.create_data_stream:
name: logs-foobar
body:
timestamp_field: "@timestamp"
- is_true: acknowledged
- do:
index:
index: logs-foobar
refresh: true
body: { foo: bar }
- match: {_index: logs-foobar-000001}
- do:
search:
index: logs-foobar
body: { query: { match_all: {} } }
- length: { hits.hits: 1 }
- match: { hits.hits.0._index: logs-foobar-000001 }
- match: { hits.hits.0._source.foo: 'bar' }
- do:
catch: bad_request
indices.delete:
index: logs-foobar
- do:
catch: bad_request
indices.get:
index: logs-foobar
- do:
catch: bad_request
indices.put_settings:
index: logs-foobar
body:
number_of_replicas: 1
- do:
catch: bad_request
indices.put_mapping:
index: logs-foobar
body:
properties:
baz:
type: keyword
- do:
indices.create:
index: logs-foobarbaz
- do:
catch: bad_request
indices.close:
index: logs-*
- do:
indices.delete_data_stream:
name: logs-foobar
- is_true: acknowledged

2
rest-api-spec/src/main/resources/rest-api-spec/test/mget/14_alias_to_multiple_indices.yml
View File

@ -42,4 +42,4 @@
- match: { docs.1._type: null }
- match: { docs.1._id: "2" }
- match: { docs.1.error.root_cause.0.type: "illegal_argument_exception" }
- match: { docs.1.error.root_cause.0.reason: "/Alias.\\[test_two_and_three\\].has.more.than.one.indices.associated.with.it.\\[\\[test_[23]{1},.test_[23]{1}\\]\\],.can't.execute.a.single.index.op/" }
- match: { docs.1.error.root_cause.0.reason: "/[aA]lias.\\[test_two_and_three\\].has.more.than.one.indices.associated.with.it.\\[\\[test_[23]{1},.test_[23]{1}\\]\\],.can't.execute.a.single.index.op/" }

2
rest-api-spec/src/main/resources/rest-api-spec/test/mget/21_alias_to_multiple_indices_with_types.yml
View File

@ -39,4 +39,4 @@
- match: { docs.1._type: test }
- match: { docs.1._id: "2" }
- match: { docs.1.error.root_cause.0.type: "illegal_argument_exception" }
- match: { docs.1.error.root_cause.0.reason: "/Alias.\\[test_two_and_three\\].has.more.than.one.indices.associated.with.it.\\[\\[test_[23]{1},.test_[23]{1}\\]\\],.can't.execute.a.single.index.op/" }
- match: { docs.1.error.root_cause.0.reason: "/[aA]lias.\\[test_two_and_three\\].has.more.than.one.indices.associated.with.it.\\[\\[test_[23]{1},.test_[23]{1}\\]\\],.can't.execute.a.single.index.op/" }

9
server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java
View File

@ -181,13 +181,18 @@ public class DeleteDataStreamAction extends ActionType<AcknowledgedResponse> {
backingIndicesToRemove.addAll(dataStream.getIndices());
dataStreamsToRemove.add(dataStreamName);
}
currentState = deleteIndexService.deleteIndices(currentState, backingIndicesToRemove);
// first delete the data streams and then the indices:
// (this to avoid data stream validation from failing when deleting an index that is part of a data stream
// without updating the data stream)
// TODO: change order when delete index api also updates the data stream the index to be removed is member of
Metadata.Builder metadata = Metadata.builder(currentState.metadata());
for (String ds : dataStreamsToRemove) {
logger.info("removing data stream [{}]", ds);
metadata.removeDataStream(ds);
}
return ClusterState.builder(currentState).metadata(metadata).build();
currentState = ClusterState.builder(currentState).metadata(metadata).build();
return deleteIndexService.deleteIndices(currentState, backingIndicesToRemove);
}
@Override

4
server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsAction.java
View File

@ -107,6 +107,10 @@ public class GetDataStreamsAction extends ActionType<GetDataStreamsAction.Respon
this(in.readList(DataStream::new));
}
public List<DataStream> getDataStreams() {
return dataStreams;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeList(dataStreams);

3
server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshRequest.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.action.admin.indices.refresh;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.broadcast.BroadcastRequest;
import org.elasticsearch.common.io.stream.StreamInput;
@ -36,7 +37,7 @@ import java.io.IOException;
public class RefreshRequest extends BroadcastRequest<RefreshRequest> {
public RefreshRequest(String... indices) {
super(indices);
super(indices, IndicesOptions.strictIncludeDataStreamsExpandOpenAndForbidClosed());
}
public RefreshRequest(StreamInput in) throws IOException {

10
server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.CompositeIndicesRequest;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.RoutingMissingException;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.replication.ReplicatedWriteRequest;
import org.elasticsearch.action.support.replication.ReplicationRequest;
import org.elasticsearch.client.Requests;
@ -257,6 +258,15 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
return validationException;
}
@Override
public IndicesOptions indicesOptions() {
if (opType == OpType.CREATE) {
return IndicesOptions.strictSingleIndexIncludeDataStreamNoExpandForbidClosed();
} else {
return IndicesOptions.strictSingleIndexNoExpandForbidClosed();
}
}
/**
* The content type. This will be used when generating a document from user provided objects like Maps and when parsing the
* source at index time

2
server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
View File

@ -60,7 +60,7 @@ public class MultiSearchRequest extends ActionRequest implements CompositeIndice
private int maxConcurrentSearchRequests = 0;
private final List<SearchRequest> requests = new ArrayList<>();
private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled();
private IndicesOptions indicesOptions = IndicesOptions.strictIncludeDataStreamsExpandOpenAndForbidClosedIgnoreThrottled();
public MultiSearchRequest() {}

3
server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
View File

@ -95,7 +95,8 @@ public class SearchRequest extends ActionRequest implements IndicesRequest.Repla
private boolean ccsMinimizeRoundtrips = true;
public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled();
public static final IndicesOptions DEFAULT_INDICES_OPTIONS =
IndicesOptions.strictIncludeDataStreamsExpandOpenAndForbidClosedIgnoreThrottled();
private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS;

83
server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
View File

@ -18,7 +18,6 @@
*/
package org.elasticsearch.action.support;
import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -103,7 +102,8 @@ public class IndicesOptions implements ToXContentFragment {
ALLOW_NO_INDICES,
FORBID_ALIASES_TO_MULTIPLE_INDICES,
FORBID_CLOSED_INDICES,
IGNORE_THROTTLED;
IGNORE_THROTTLED,
INCLUDE_DATA_STREAMS;
public static final EnumSet<Option> NONE = EnumSet.noneOf(Option.class);
}
@ -113,6 +113,9 @@ public class IndicesOptions implements ToXContentFragment {
public static final IndicesOptions LENIENT_EXPAND_OPEN =
new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.IGNORE_UNAVAILABLE),
EnumSet.of(WildcardStates.OPEN));
public static final IndicesOptions LENIENT_INCLUDE_DATA_STREAMS_EXPAND_OPEN =
new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.IGNORE_UNAVAILABLE, Option.INCLUDE_DATA_STREAMS),
EnumSet.of(WildcardStates.OPEN));
public static final IndicesOptions LENIENT_EXPAND_OPEN_HIDDEN =
new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.IGNORE_UNAVAILABLE),
EnumSet.of(WildcardStates.OPEN, WildcardStates.HIDDEN));
@ -132,13 +135,20 @@ public class IndicesOptions implements ToXContentFragment {
public static final IndicesOptions STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED =
new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.FORBID_CLOSED_INDICES),
EnumSet.of(WildcardStates.OPEN, WildcardStates.HIDDEN));
public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED =
new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.FORBID_CLOSED_INDICES, Option.IGNORE_THROTTLED),
EnumSet.of(WildcardStates.OPEN));
public static final IndicesOptions STRICT_INCLUDE_DATA_STREAMS_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED =
new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.FORBID_CLOSED_INDICES, Option.IGNORE_THROTTLED,
Option.INCLUDE_DATA_STREAMS), EnumSet.of(WildcardStates.OPEN));
public static final IndicesOptions STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED =
new IndicesOptions(EnumSet.of(Option.FORBID_ALIASES_TO_MULTIPLE_INDICES, Option.FORBID_CLOSED_INDICES),
EnumSet.noneOf(WildcardStates.class));
public static final IndicesOptions STRICT_INCLUDE_DATA_STREAMS_EXPAND_OPEN_FORBID_CLOSED =
new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.INCLUDE_DATA_STREAMS, Option.FORBID_CLOSED_INDICES),
EnumSet.of(WildcardStates.OPEN));
public static final IndicesOptions STRICT_SINGLE_INDEX_INCLUDE_DATA_STREAMS_NO_EXPAND_FORBID_CLOSED =
new IndicesOptions(EnumSet.of(Option.FORBID_ALIASES_TO_MULTIPLE_INDICES, Option.FORBID_CLOSED_INDICES,
Option.INCLUDE_DATA_STREAMS), EnumSet.noneOf(WildcardStates.class));
private final EnumSet<Option> options;
private final EnumSet<WildcardStates> expandWildcards;
@ -227,6 +237,13 @@ public class IndicesOptions implements ToXContentFragment {
return EnumSet.copyOf(expandWildcards);
}
/**
* @return Whether to include data streams when resolving index expressions to concrete indices.
*/
public boolean includeDataStreams() {
return options.contains(Option.INCLUDE_DATA_STREAMS);
}
public void writeIndicesOptions(StreamOutput out) throws IOException {
EnumSet<Option> options = this.options;
// never write this out to a pre 6.6 version
@ -234,6 +251,10 @@ public class IndicesOptions implements ToXContentFragment {
options = EnumSet.copyOf(options);
options.remove(Option.IGNORE_THROTTLED);
}
if (out.getVersion().before(Version.V_7_8_0) && options.contains(Option.INCLUDE_DATA_STREAMS)) {
options = EnumSet.copyOf(options);
options.remove(Option.INCLUDE_DATA_STREAMS);
}
out.writeEnumSet(options);
if (out.getVersion().before(Version.V_7_7_0) && expandWildcards.contains(WildcardStates.HIDDEN)) {
final EnumSet<WildcardStates> states = EnumSet.copyOf(expandWildcards);
@ -261,14 +282,15 @@ public class IndicesOptions implements ToXContentFragment {
public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices,
boolean expandToClosedIndices, boolean expandToHiddenIndices) {
return fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices, expandToHiddenIndices, true,
false, false, false);
false, false, false, false);
}
public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices,
boolean expandToClosedIndices, IndicesOptions defaultOptions) {
return fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices,
defaultOptions.expandWildcardsHidden(), defaultOptions.allowAliasesToMultipleIndices(),
defaultOptions.forbidClosedIndices(), defaultOptions.ignoreAliases(), defaultOptions.ignoreThrottled());
defaultOptions.forbidClosedIndices(), defaultOptions.ignoreAliases(), defaultOptions.ignoreThrottled(),
defaultOptions.includeDataStreams());
}
public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices,
@ -276,13 +298,13 @@ public class IndicesOptions implements ToXContentFragment {
boolean forbidClosedIndices, boolean ignoreAliases,
boolean ignoreThrottled) {
return fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices, false,
allowAliasesToMultipleIndices, forbidClosedIndices, ignoreAliases, ignoreThrottled);
allowAliasesToMultipleIndices, forbidClosedIndices, ignoreAliases, ignoreThrottled, false);
}
public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices,
boolean expandToClosedIndices, boolean expandToHiddenIndices,
boolean allowAliasesToMultipleIndices, boolean forbidClosedIndices, boolean ignoreAliases,
boolean ignoreThrottled) {
boolean ignoreThrottled, boolean includeDataStreams) {
final EnumSet<Option> opts = EnumSet.noneOf(Option.class);
final EnumSet<WildcardStates> wildcards = EnumSet.noneOf(WildcardStates.class);
@ -313,6 +335,9 @@ public class IndicesOptions implements ToXContentFragment {
if (ignoreThrottled) {
opts.add(Option.IGNORE_THROTTLED);
}
if (includeDataStreams) {
opts.add(Option.INCLUDE_DATA_STREAMS);
}
return new IndicesOptions(opts, wildcards);
}
@ -364,8 +389,8 @@ public class IndicesOptions implements ToXContentFragment {
defaultSettings.allowAliasesToMultipleIndices(),
defaultSettings.forbidClosedIndices(),
defaultSettings.ignoreAliases(),
nodeBooleanValue(ignoreThrottled, "ignore_throttled", defaultSettings.ignoreThrottled())
);
nodeBooleanValue(ignoreThrottled, "ignore_throttled", defaultSettings.ignoreThrottled()),
defaultSettings.includeDataStreams());
}
@Override
@ -400,11 +425,21 @@ public class IndicesOptions implements ToXContentFragment {
/**
* @return indices options that requires every specified index to exist, expands wildcards only to open indices,
* allows that no indices are resolved from wildcard expressions (not returning an error) and forbids the
* use of closed indices by throwing an error and ignores indices that are throttled.
* allows that no indices are resolved from wildcard expressions (not returning an error),
* supports data streams and forbids the use of closed indices by throwing an error.
*/
public static IndicesOptions strictExpandOpenAndForbidClosedIgnoreThrottled() {
return STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED;
public static IndicesOptions strictIncludeDataStreamsExpandOpenAndForbidClosed() {
return STRICT_INCLUDE_DATA_STREAMS_EXPAND_OPEN_FORBID_CLOSED;
}
/**
* @return indices options that requires every specified index to exist, expands wildcards only to open indices,
* allows that no indices are resolved from wildcard expressions (not returning an error),
* supports data streams and forbids the use of closed indices by throwing an error and
* ignores indices that are throttled.
*/
public static IndicesOptions strictIncludeDataStreamsExpandOpenAndForbidClosedIgnoreThrottled() {
return STRICT_INCLUDE_DATA_STREAMS_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED;
}
/**
@ -431,6 +466,14 @@ public class IndicesOptions implements ToXContentFragment {
return STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED;
}
/**
* @return indices option that requires each specified index or alias to exist, doesn't expand wildcards,
* supports data streams and throws error if any of the aliases resolves to multiple indices.
*/
public static IndicesOptions strictSingleIndexIncludeDataStreamNoExpandForbidClosed() {
return STRICT_SINGLE_INDEX_INCLUDE_DATA_STREAMS_NO_EXPAND_FORBID_CLOSED;
}
/**
* @return indices options that ignores unavailable indices, expands wildcards only to open indices and
* allows that no indices are resolved from wildcard expressions (not returning an error).
@ -439,6 +482,15 @@ public class IndicesOptions implements ToXContentFragment {
return LENIENT_EXPAND_OPEN;
}
/**
* @return indices options that ignores unavailable indices, expands wildcards only to open indices,
* supports data streams and allows that no indices are resolved from wildcard expressions
* (not returning an error).
*/
public static IndicesOptions lenientIncludeDataStreamsExpandOpen() {
return LENIENT_INCLUDE_DATA_STREAMS_EXPAND_OPEN;
}
/**
* @return indices options that ignores unavailable indices, expands wildcards to open and hidden indices, and
* allows that no indices are resolved from wildcard expressions (not returning an error).
@ -495,6 +547,7 @@ public class IndicesOptions implements ToXContentFragment {
", forbid_closed_indices=" + forbidClosedIndices() +
", ignore_aliases=" + ignoreAliases() +
", ignore_throttled=" + ignoreThrottled() +
", include_data_streams=" + includeDataStreams() +
']';
}
}

15
server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java
View File

@ -30,9 +30,11 @@ import org.elasticsearch.action.admin.indices.rollover.Condition;
import org.elasticsearch.cli.EnvironmentAwareCommand;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.cli.UserException;
import org.elasticsearch.cluster.ClusterModule;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.Diff;
import org.elasticsearch.cluster.metadata.DataStreamMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -51,7 +53,6 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Map;
import java.util.Objects;
@ -73,14 +74,22 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
"cluster state is empty, cluster has never been bootstrapped?";
// fake the registry here, as command-line tools are not loading plugins, and ensure that it preserves the parsed XContent
public static final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(Collections.emptyList()) {
public static final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(ClusterModule.getNamedXWriteables()) {
@SuppressWarnings("unchecked")
@Override
public <T, C> T parseNamedObject(Class<T> categoryClass, String name, XContentParser parser, C context) throws IOException {
// Currently, two unknown top-level objects are present
if (Metadata.Custom.class.isAssignableFrom(categoryClass)) {
return (T) new UnknownMetadataCustom(name, parser.mapOrdered());
if (DataStreamMetadata.TYPE.equals(name)) {
// DataStreamMetadata is used inside Metadata class for validation purposes and building the indicesLookup,
// therefor even es node commands need to be able to parse it.
return super.parseNamedObject(categoryClass, name, parser, context);
// TODO: Try to parse other named objects (e.g. stored scripts, ingest pipelines) that are part of core es as well?
// Note that supporting PersistentTasksCustomMetadata is trickier, because PersistentTaskParams is a named object too.
} else {
return (T) new UnknownMetadataCustom(name, parser.mapOrdered());
}
}
if (Condition.class.isAssignableFrom(categoryClass)) {
// The parsing for conditions is a bit weird as these represent JSON primitives (strings or numbers)

40
server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java
View File

@ -32,6 +32,7 @@ import java.util.Objects;
import java.util.stream.Collectors;
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_HIDDEN_SETTING;
import static org.elasticsearch.common.collect.List.copyOf;
/**
* An index abstraction is a reference to one or more concrete indices.
@ -258,4 +259,43 @@ public interface IndexAbstraction {
return (Objects.isNull(idxMetas) || idxMetas.isEmpty()) == false;
}
}
class DataStream implements IndexAbstraction {
private final org.elasticsearch.cluster.metadata.DataStream dataStream;
private final List<IndexMetadata> dataStreamIndices;
private final IndexMetadata writeIndex;
public DataStream(org.elasticsearch.cluster.metadata.DataStream dataStream,
List<IndexMetadata> dataStreamIndices, IndexMetadata writeIndex) {
this.dataStream = dataStream;
this.dataStreamIndices = copyOf(dataStreamIndices);
this.writeIndex = writeIndex;
assert dataStreamIndices.contains(writeIndex);
}
@Override
public String getName() {
return dataStream.getName();
}
@Override
public Type getType() {
return Type.DATA_STREAM;
}
@Override
public List<IndexMetadata> getIndices() {
return dataStreamIndices;
}
public IndexMetadata getWriteIndex() {
return writeIndex;
}
@Override
public boolean isHidden() {
return false;
}
}
}

99
server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
View File

@ -55,6 +55,7 @@ import java.util.SortedMap;
import java.util.Spliterators;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import static java.util.Collections.unmodifiableList;
@ -204,6 +205,9 @@ public class IndexNameExpressionResolver {
} else {
continue;
}
} else if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM &&
context.getOptions().includeDataStreams() == false) {
throw dataStreamsNotSupportedException(expression);
}
if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS && context.isResolveToWriteIndex()) {
@ -216,6 +220,11 @@ public class IndexNameExpressionResolver {
if (addIndex(writeIndex, context)) {
concreteIndices.add(writeIndex.getIndex());
}
} else if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM && context.isResolveToWriteIndex()) {
IndexMetadata writeIndex = indexAbstraction.getWriteIndex();
if (addIndex(writeIndex, context)) {
concreteIndices.add(writeIndex.getIndex());
}
} else {
if (indexAbstraction.getIndices().size() > 1 && !options.allowAliasesToMultipleIndices()) {
String[] indexNames = new String[indexAbstraction.getIndices().size()];
@ -223,8 +232,9 @@ public class IndexNameExpressionResolver {
for (IndexMetadata indexMetadata : indexAbstraction.getIndices()) {
indexNames[i++] = indexMetadata.getIndex().getName();
}
throw new IllegalArgumentException("Alias [" + expression + "] has more than one indices associated with it [" +
Arrays.toString(indexNames) + "], can't execute a single index op");
throw new IllegalArgumentException(indexAbstraction.getType().getDisplayName() + " [" + expression +
"] has more than one indices associated with it [" + Arrays.toString(indexNames) +
"], can't execute a single index op");
}
for (IndexMetadata index : indexAbstraction.getIndices()) {
@ -264,6 +274,11 @@ public class IndexNameExpressionResolver {
"alias, specify the corresponding concrete indices instead.");
}
private static IllegalArgumentException dataStreamsNotSupportedException(String expression) {
return new IllegalArgumentException("The provided expression [" + expression + "] matches a " +
"data stream, specify the corresponding concrete indices instead.");
}
/**
* Utility method that allows to resolve an index expression to its corresponding single concrete index.
* Callers should make sure they provide proper {@link org.elasticsearch.action.support.IndicesOptions}
@ -347,7 +362,7 @@ public class IndexNameExpressionResolver {
* Resolve an array of expressions to the set of indices and aliases that these expressions match.
*/
public Set<String> resolveExpressions(ClusterState state, String... expressions) {
Context context = new Context(state, IndicesOptions.lenientExpandOpen(), true, false);
Context context = new Context(state, IndicesOptions.lenientIncludeDataStreamsExpandOpen(), true, false);
List<String> resolvedExpressions = Arrays.asList(expressions);
for (ExpressionResolver expressionResolver : expressionResolvers) {
resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions);
@ -440,8 +455,8 @@ public class IndexNameExpressionResolver {
* @return routing values grouped by concrete index
*/
public Map<String, Set<String>> resolveSearchRouting(ClusterState state, @Nullable String routing, String... expressions) {
List<String> resolvedExpressions = expressions != null ? Arrays.asList(expressions) : Collections.<String>emptyList();
Context context = new Context(state, IndicesOptions.lenientExpandOpen());
List<String> resolvedExpressions = expressions != null ? Arrays.asList(expressions) : Collections.emptyList();
Context context = new Context(state, IndicesOptions.lenientIncludeDataStreamsExpandOpen());
for (ExpressionResolver expressionResolver : expressionResolvers) {
resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions);
}
@ -680,6 +695,9 @@ public class IndexNameExpressionResolver {
}
if (isEmptyOrTrivialWildcard(expressions)) {
if (options.includeDataStreams() == false && metadata.dataStreams().isEmpty() == false) {
throw dataStreamsNotSupportedException(expressions.toString());
}
return resolveEmptyOrTrivialWildcard(options, metadata);
}
@ -730,6 +748,9 @@ public class IndexNameExpressionResolver {
throw indexNotFoundException(expression);
} else if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS && options.ignoreAliases()) {
throw aliasesNotSupportedException(expression);
} else if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM &&
options.includeDataStreams() == false) {
throw dataStreamsNotSupportedException(expression);
}
}
if (add) {
@ -770,9 +791,20 @@ public class IndexNameExpressionResolver {
private static boolean aliasOrIndexExists(IndicesOptions options, Metadata metadata, String expression) {
IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(expression);
if (indexAbstraction == null) {
return false;
}
//treat aliases as unavailable indices when ignoreAliases is set to true (e.g. delete index and update aliases api)
return indexAbstraction != null && (options.ignoreAliases() == false ||
indexAbstraction.getType() != IndexAbstraction.Type.ALIAS);
if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS && options.ignoreAliases()) {
return false;
}
if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM && options.includeDataStreams() == false) {
return false;
}
return true;
}
private static IndexNotFoundException indexNotFoundException(String expression) {
@ -798,14 +830,7 @@ public class IndexNameExpressionResolver {
public static Map<String, IndexAbstraction> matches(Context context, Metadata metadata, String expression) {
if (Regex.isMatchAllPattern(expression)) {
// Can only happen if the expressions was initially: '-*'
if (context.getOptions().ignoreAliases()) {
return metadata.getIndicesLookup().entrySet().stream()
.filter(e -> e.getValue().getType() != IndexAbstraction.Type.ALIAS)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
} else {
return metadata.getIndicesLookup();
}
return filterIndicesLookup(metadata.getIndicesLookup(), null, expression, context.getOptions());
} else if (expression.indexOf("*") == expression.length() - 1) {
return suffixWildcard(context, metadata, expression);
} else {
@ -820,22 +845,42 @@ public class IndexNameExpressionResolver {
toPrefixCharArr[toPrefixCharArr.length - 1]++;
String toPrefix = new String(toPrefixCharArr);
SortedMap<String, IndexAbstraction> subMap = metadata.getIndicesLookup().subMap(fromPrefix, toPrefix);
if (context.getOptions().ignoreAliases()) {
return subMap.entrySet().stream()
.filter(entry -> entry.getValue().getType() != IndexAbstraction.Type.ALIAS)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}
return subMap;
return filterIndicesLookup(subMap, null, expression, context.getOptions());
}
private static Map<String, IndexAbstraction> otherWildcard(Context context, Metadata metadata, String expression) {
final String pattern = expression;
return metadata.getIndicesLookup()
.entrySet()
.stream()
.filter(e -> context.getOptions().ignoreAliases() == false || e.getValue().getType() != IndexAbstraction.Type.ALIAS)
.filter(e -> Regex.simpleMatch(pattern, e.getKey()))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
return filterIndicesLookup(metadata.getIndicesLookup(), e -> Regex.simpleMatch(pattern, e.getKey()),
expression, context.getOptions());
}
private static Map<String, IndexAbstraction> filterIndicesLookup(SortedMap<String, IndexAbstraction> indicesLookup,
Predicate<? super Map.Entry<String, IndexAbstraction>> filter,
String expression,
IndicesOptions options) {
boolean shouldConsumeStream = false;
Stream<Map.Entry<String, IndexAbstraction>> stream = indicesLookup.entrySet().stream();
if (options.ignoreAliases()) {
shouldConsumeStream = true;
stream = stream.filter(e -> e.getValue().getType() != IndexAbstraction.Type.ALIAS);
}
if (filter != null) {
shouldConsumeStream = true;
stream = stream.filter(filter);
}
if (options.includeDataStreams() == false) {
shouldConsumeStream = true;
stream = stream.peek(e -> {
if (e.getValue().getType() == IndexAbstraction.Type.DATA_STREAM) {
throw dataStreamsNotSupportedException(expression);
}
});
}
if (shouldConsumeStream) {
return stream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
} else {
return indicesLookup;
}
}
private static Set<String> expand(Context context, IndexMetadata.State excludeState, Map<String, IndexAbstraction> matches,

38
server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java
View File

@ -1424,16 +1424,16 @@ public class Metadata implements Iterable<IndexMetadata>, Diffable<Metadata>, To
}
private SortedMap<String, IndexAbstraction> buildIndicesLookup() {
SortedMap<String, IndexAbstraction> aliasAndIndexLookup = new TreeMap<>();
SortedMap<String, IndexAbstraction> indicesLookup = new TreeMap<>();
for (ObjectCursor<IndexMetadata> cursor : indices.values()) {
IndexMetadata indexMetadata = cursor.value;
IndexAbstraction existing =
aliasAndIndexLookup.put(indexMetadata.getIndex().getName(), new IndexAbstraction.Index(indexMetadata));
indicesLookup.put(indexMetadata.getIndex().getName(), new IndexAbstraction.Index(indexMetadata));
assert existing == null : "duplicate for " + indexMetadata.getIndex();
for (ObjectObjectCursor<String, AliasMetadata> aliasCursor : indexMetadata.getAliases()) {
AliasMetadata aliasMetadata = aliasCursor.value;
aliasAndIndexLookup.compute(aliasMetadata.getAlias(), (aliasName, alias) -> {
indicesLookup.compute(aliasMetadata.getAlias(), (aliasName, alias) -> {
if (alias == null) {
return new IndexAbstraction.Alias(aliasMetadata, indexMetadata);
} else {
@ -1445,21 +1445,39 @@ public class Metadata implements Iterable<IndexMetadata>, Diffable<Metadata>, To
}
}
aliasAndIndexLookup.values().stream()
DataStreamMetadata dataStreamMetadata = (DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE);
// If there are no indices then it doesn't make sense to to add data streams to indicesLookup,
// since there no concrete indices that a data stream can point to.
// (This occurs when only Metadata is read from disk.)
if (dataStreamMetadata != null && indices.size() > 0) {
for (Map.Entry<String, DataStream> entry : dataStreamMetadata.dataStreams().entrySet()) {
DataStream dataStream = entry.getValue();
List<IndexMetadata> backingIndices = dataStream.getIndices().stream()
.map(index -> indices.get(index.getName()))
.collect(Collectors.toList());
assert backingIndices.isEmpty() == false;
assert backingIndices.contains(null) == false;
IndexMetadata writeIndex = backingIndices.get(backingIndices.size() - 1);
IndexAbstraction existing = indicesLookup.put(dataStream.getName(),
new IndexAbstraction.DataStream(dataStream, backingIndices, writeIndex));
if (existing != null) {
throw new IllegalStateException("data stream [" + dataStream.getName() +
"] conflicts with existing " + existing.getType().getDisplayName() + " [" + existing.getName() + "]");
}
}
}
indicesLookup.values().stream()
.filter(aliasOrIndex -> aliasOrIndex.getType() == IndexAbstraction.Type.ALIAS)
.forEach(alias -> ((IndexAbstraction.Alias) alias).computeAndValidateAliasProperties());
return aliasAndIndexLookup;
return indicesLookup;
}
private void validateDataStreams(SortedMap<String, IndexAbstraction> indicesLookup) {
DataStreamMetadata dsMetadata = (DataStreamMetadata) customs.get(DataStreamMetadata.TYPE);
if (dsMetadata != null) {
for (DataStream ds : dsMetadata.dataStreams().values()) {
IndexAbstraction existing = indicesLookup.get(ds.getName());
if (existing != null && existing.getType() != IndexAbstraction.Type.DATA_STREAM) {
throw new IllegalStateException("data stream [" + ds.getName() + "] conflicts with existing index or alias");
}
SortedMap<String, IndexAbstraction> potentialConflicts =
indicesLookup.subMap(ds.getName() + "-", ds.getName() + "."); // '.' is the char after '-'
if (potentialConflicts.size() != 0) {

10
server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java
View File

@ -28,12 +28,13 @@ import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
import org.elasticsearch.common.collect.List;
import org.elasticsearch.common.collect.Map;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.AbstractWireSerializingTestCase;
import java.util.Collections;
import static org.elasticsearch.cluster.DataStreamTestHelper.createFirstBackingIndex;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
@ -86,9 +87,10 @@ public class CreateDataStreamRequestTests extends AbstractWireSerializingTestCas
public void testCreateDuplicateDataStream() throws Exception {
final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService();
final String dataStreamName = "my-data-stream";
DataStream existingDataStream = new DataStream(dataStreamName, "timestamp", Collections.emptyList());
IndexMetadata idx = createFirstBackingIndex(dataStreamName).build();
DataStream existingDataStream = new DataStream(dataStreamName, "timestamp", List.of(idx.getIndex()));
ClusterState cs = ClusterState.builder(new ClusterName("_name"))
.metadata(Metadata.builder().dataStreams(Collections.singletonMap(dataStreamName, existingDataStream)).build()).build();
.metadata(Metadata.builder().dataStreams(Map.of(dataStreamName, existingDataStream)).build()).build();
CreateDataStreamAction.Request req = new CreateDataStreamAction.Request(dataStreamName);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,

15
server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java
View File

@ -22,13 +22,14 @@ import org.elasticsearch.ResourceNotFoundException;
import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction.Request;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.DataStreamTestHelper;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.common.collect.Map;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.test.AbstractWireSerializingTestCase;
import java.util.Collections;
import java.util.List;
import static org.hamcrest.Matchers.containsString;
@ -63,7 +64,9 @@ public class GetDataStreamsRequestTests extends AbstractWireSerializingTestCase<
public void testGetDataStream() {
final String dataStreamName = "my-data-stream";
DataStream existingDataStream = new DataStream(dataStreamName, "timestamp", Collections.emptyList());
IndexMetadata idx = DataStreamTestHelper.createFirstBackingIndex(dataStreamName).build();
DataStream existingDataStream =
new DataStream(dataStreamName, "timestamp", org.elasticsearch.common.collect.List.of(idx.getIndex()));
ClusterState cs = ClusterState.builder(new ClusterName("_name"))
.metadata(Metadata.builder().dataStreams(Map.of(dataStreamName, existingDataStream)).build()).build();
GetDataStreamsAction.Request req = new GetDataStreamsAction.Request(dataStreamName);
@ -74,8 +77,11 @@ public class GetDataStreamsRequestTests extends AbstractWireSerializingTestCase<
public void testGetDataStreamsWithWildcards() {
final String[] dataStreamNames = {"my-data-stream", "another-data-stream"};
DataStream ds1 = new DataStream(dataStreamNames[0], "timestamp", Collections.emptyList());
DataStream ds2 = new DataStream(dataStreamNames[1], "timestamp", Collections.emptyList());
IndexMetadata idx1 = DataStreamTestHelper.createFirstBackingIndex(dataStreamNames[0]).build();
IndexMetadata idx2 = DataStreamTestHelper.createFirstBackingIndex(dataStreamNames[1]).build();
DataStream ds1 = new DataStream(dataStreamNames[0], "timestamp", org.elasticsearch.common.collect.List.of(idx1.getIndex()));
DataStream ds2 = new DataStream(dataStreamNames[1], "timestamp", org.elasticsearch.common.collect.List.of(idx2.getIndex()));
ClusterState cs = ClusterState.builder(new ClusterName("_name"))
.metadata(Metadata.builder().dataStreams(
Map.of(dataStreamNames[0], ds1, dataStreamNames[1], ds2)).build())
@ -111,4 +117,5 @@ public class GetDataStreamsRequestTests extends AbstractWireSerializingTestCase<
() -> GetDataStreamsAction.TransportAction.getDataStreams(cs, req));
assertThat(e.getMessage(), containsString("data_stream matching [" + dataStreamName + "] not found"));
}
}

23
server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
View File

@ -329,21 +329,21 @@ public class MultiSearchRequestTests extends ESTestCase {
public void testWritingExpandWildcards() throws IOException {
assertExpandWildcardsValue(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, true, true, randomBoolean(),
randomBoolean(), randomBoolean(), randomBoolean()), "all");
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()), "all");
assertExpandWildcardsValue(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, true, false, randomBoolean(),
randomBoolean(), randomBoolean(), randomBoolean()), "open,closed");
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()), "open,closed");
assertExpandWildcardsValue(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false, true, randomBoolean(),
randomBoolean(), randomBoolean(), randomBoolean()), "open,hidden");
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()), "open,hidden");
assertExpandWildcardsValue(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false, false, randomBoolean(),
randomBoolean(), randomBoolean(), randomBoolean()), "open");
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()), "open");
assertExpandWildcardsValue(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), false, true, true, randomBoolean(),
randomBoolean(), randomBoolean(), randomBoolean()), "closed,hidden");
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()), "closed,hidden");
assertExpandWildcardsValue(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), false, true, false, randomBoolean(),
randomBoolean(), randomBoolean(), randomBoolean()), "closed");
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()), "closed");
assertExpandWildcardsValue(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), false, false, true, randomBoolean(),
randomBoolean(), randomBoolean(), randomBoolean()), "hidden");
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()), "hidden");
assertExpandWildcardsValue(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), false, false, false, randomBoolean(),
randomBoolean(), randomBoolean(), randomBoolean()), "none");
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()), "none");
}
private void assertExpandWildcardsValue(IndicesOptions options, String expectedValue) throws IOException {
@ -402,9 +402,10 @@ public class MultiSearchRequestTests extends ESTestCase {
IndicesOptions randomlyGenerated = searchRequest.indicesOptions();
IndicesOptions msearchDefault = SearchRequest.DEFAULT_INDICES_OPTIONS;
searchRequest.indicesOptions(IndicesOptions.fromOptions(
randomlyGenerated.ignoreUnavailable(), randomlyGenerated.allowNoIndices(), randomlyGenerated.expandWildcardsOpen(),
randomlyGenerated.expandWildcardsClosed(), msearchDefault.allowAliasesToMultipleIndices(),
msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases(), msearchDefault.ignoreThrottled()
randomlyGenerated.ignoreUnavailable(), randomlyGenerated.allowNoIndices(), randomlyGenerated.expandWildcardsOpen(),
randomlyGenerated.expandWildcardsClosed(), msearchDefault.expandWildcardsHidden(),
msearchDefault.allowAliasesToMultipleIndices(), msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases(),
msearchDefault.ignoreThrottled(), msearchDefault.includeDataStreams()
));
request.add(searchRequest);

28
server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
View File

@ -57,7 +57,7 @@ public class IndicesOptionsTests extends ESTestCase {
Version version = randomVersionBetween(random(), Version.V_7_0_0, null);
IndicesOptions indicesOptions = IndicesOptions.fromOptions(
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
randomBoolean(), randomBoolean());
randomBoolean(), randomBoolean(), randomBoolean());
BytesStreamOutput output = new BytesStreamOutput();
output.setVersion(version);
@ -81,6 +81,11 @@ public class IndicesOptionsTests extends ESTestCase {
assertThat(indicesOptions2.allowAliasesToMultipleIndices(), equalTo(indicesOptions.allowAliasesToMultipleIndices()));
assertEquals(indicesOptions2.ignoreAliases(), indicesOptions.ignoreAliases());
if (version.before(Version.V_7_8_0)) {
assertThat(indicesOptions2.includeDataStreams(), is(false));
} else {
assertThat(indicesOptions2.includeDataStreams(), equalTo(indicesOptions.includeDataStreams()));
}
}
}
@ -126,10 +131,11 @@ public class IndicesOptionsTests extends ESTestCase {
final boolean forbidClosedIndices = randomBoolean();
final boolean ignoreAliases = randomBoolean();
final boolean ignoreThrottled = randomBoolean();
final boolean includeDataStreams = randomBoolean();
IndicesOptions indicesOptions = IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices,
expandToClosedIndices, expandToHiddenIndices, allowAliasesToMultipleIndices, forbidClosedIndices, ignoreAliases,
ignoreThrottled);
ignoreThrottled, includeDataStreams);
assertThat(indicesOptions.ignoreUnavailable(), equalTo(ignoreUnavailable));
assertThat(indicesOptions.allowNoIndices(), equalTo(allowNoIndices));
@ -141,6 +147,7 @@ public class IndicesOptionsTests extends ESTestCase {
assertThat(indicesOptions.forbidClosedIndices(), equalTo(forbidClosedIndices));
assertEquals(ignoreAliases, indicesOptions.ignoreAliases());
assertEquals(ignoreThrottled, indicesOptions.ignoreThrottled());
assertThat(indicesOptions.includeDataStreams(), equalTo(includeDataStreams));
}
public void testFromOptionsWithDefaultOptions() {
@ -150,7 +157,7 @@ public class IndicesOptionsTests extends ESTestCase {
boolean expandToClosedIndices = randomBoolean();
IndicesOptions defaultOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
IndicesOptions indicesOptions = IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices,
expandToClosedIndices, defaultOptions);
@ -163,6 +170,7 @@ public class IndicesOptionsTests extends ESTestCase {
assertEquals(defaultOptions.allowAliasesToMultipleIndices(), indicesOptions.allowAliasesToMultipleIndices());
assertEquals(defaultOptions.forbidClosedIndices(), indicesOptions.forbidClosedIndices());
assertEquals(defaultOptions.ignoreAliases(), indicesOptions.ignoreAliases());
assertEquals(defaultOptions.includeDataStreams(), indicesOptions.includeDataStreams());
}
public void testFromParameters() {
@ -199,7 +207,7 @@ public class IndicesOptionsTests extends ESTestCase {
String allowNoIndicesString = Boolean.toString(allowNoIndices);
IndicesOptions defaultOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
IndicesOptions updatedOptions = IndicesOptions.fromParameters(expandWildcardsString, ignoreUnavailableString,
allowNoIndicesString, ignoreThrottled, defaultOptions);
@ -212,16 +220,17 @@ public class IndicesOptionsTests extends ESTestCase {
assertEquals(defaultOptions.allowAliasesToMultipleIndices(), updatedOptions.allowAliasesToMultipleIndices());
assertEquals(defaultOptions.forbidClosedIndices(), updatedOptions.forbidClosedIndices());
assertEquals(defaultOptions.ignoreAliases(), updatedOptions.ignoreAliases());
assertEquals(defaultOptions.includeDataStreams(), updatedOptions.includeDataStreams());
}
public void testEqualityAndHashCode() {
IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
EqualsHashCodeTestUtils.checkEqualsAndHashCode(indicesOptions, opts -> {
return IndicesOptions.fromOptions(opts.ignoreUnavailable(), opts.allowNoIndices(), opts.expandWildcardsOpen(),
opts.expandWildcardsClosed(), opts.expandWildcardsHidden(), opts.allowAliasesToMultipleIndices(),
opts.forbidClosedIndices(), opts.ignoreAliases(), opts.ignoreThrottled());
opts.forbidClosedIndices(), opts.ignoreAliases(), opts.ignoreThrottled(), opts.includeDataStreams());
}, opts -> {
boolean mutated = false;
boolean ignoreUnavailable = opts.ignoreUnavailable();
@ -233,6 +242,7 @@ public class IndicesOptionsTests extends ESTestCase {
boolean forbidClosed = opts.forbidClosedIndices();
boolean ignoreAliases = opts.ignoreAliases();
boolean ignoreThrottled = opts.ignoreThrottled();
boolean includeDataStreams = opts.includeDataStreams();
while (mutated == false) {
if (randomBoolean()) {
ignoreUnavailable = !ignoreUnavailable;
@ -270,9 +280,13 @@ public class IndicesOptionsTests extends ESTestCase {
ignoreThrottled = !ignoreThrottled;
mutated = true;
}
if (randomBoolean()) {
includeDataStreams = !includeDataStreams;
mutated = true;
}
}
return IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices, expandOpen, expandClosed, expandHidden,
allowAliasesToMulti, forbidClosed, ignoreAliases, ignoreThrottled);
allowAliasesToMulti, forbidClosed, ignoreAliases, ignoreThrottled, includeDataStreams);
});
}

2
server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
View File

@ -1227,7 +1227,7 @@ public class IndexAliasesIT extends ESIntegTestCase {
// And querying using a wildcard with indices options set to expand hidden
searchResponse = client().prepareSearch("alias*")
.setQuery(QueryBuilders.matchAllQuery())
.setIndicesOptions(IndicesOptions.fromOptions(false, false, true, false, true, true, true, false, false)).get();
.setIndicesOptions(IndicesOptions.fromOptions(false, false, true, false, true, true, true, false, false, true)).get();
assertHits(searchResponse.getHits(), "1", "2", "3");
// And that querying the alias with a wildcard and no expand options fails

19
server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandTests.java
View File

@ -20,10 +20,13 @@
package org.elasticsearch.cluster.coordination;
import org.elasticsearch.cluster.ClusterModule;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.cluster.metadata.IndexGraveyard;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.List;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
@ -38,6 +41,7 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.elasticsearch.cluster.DataStreamTestHelper.createFirstBackingIndex;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.not;
@ -74,6 +78,7 @@ public class ElasticsearchNodeCommandTests extends ESTestCase {
}
assertThat(loadedMetadata.clusterUUID(), not(equalTo("_na_")));
assertThat(loadedMetadata.clusterUUID(), equalTo(latestMetadata.clusterUUID()));
assertThat(loadedMetadata.dataStreams(), equalTo(latestMetadata.dataStreams()));
// make sure the index tombstones are the same too
if (hasMissingCustoms) {
@ -100,14 +105,24 @@ public class ElasticsearchNodeCommandTests extends ESTestCase {
for (int i = 0; i < numDelIndices; i++) {
graveyard.addTombstone(new Index(randomAlphaOfLength(10) + "del-idx-" + i, UUIDs.randomBase64UUID()));
}
if (randomBoolean()) {
int numDataStreams = randomIntBetween(0, 5);
for (int i = 0; i < numDataStreams; i++) {
String dataStreamName = "name" + 1;
IndexMetadata backingIndex = createFirstBackingIndex(dataStreamName).build();
mdBuilder.put(new DataStream(dataStreamName, "ts", List.of(backingIndex.getIndex())));
}
}
mdBuilder.indexGraveyard(graveyard.build());
return mdBuilder.build();
}
@Override
protected NamedXContentRegistry xContentRegistry() {
return new NamedXContentRegistry(Stream.of(ClusterModule.getNamedXWriteables().stream(), IndicesModule.getNamedXContents().stream())
return new NamedXContentRegistry(
Stream.of(ClusterModule.getNamedXWriteables().stream(), IndicesModule.getNamedXContents().stream())
.flatMap(Function.identity())
.collect(Collectors.toList()));
.collect(Collectors.toList())
);
}
}

174
server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
View File

@ -49,6 +49,7 @@ import java.util.List;
import java.util.Set;
import java.util.function.Function;
import static org.elasticsearch.cluster.DataStreamTestHelper.createBackingIndex;
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_HIDDEN_SETTING;
import static org.elasticsearch.common.util.set.Sets.newHashSet;
import static org.hamcrest.Matchers.arrayContaining;
@ -602,7 +603,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed());
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> indexNameExpressionResolver.concreteIndexNames(context, "foofoobar"));
assertThat(e.getMessage(), containsString("Alias [foofoobar] has more than one indices associated with it"));
assertThat(e.getMessage(), containsString("alias [foofoobar] has more than one indices associated with it"));
}
{
@ -610,7 +611,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed());
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> indexNameExpressionResolver.concreteIndexNames(context, "foo", "foofoobar"));
assertThat(e.getMessage(), containsString("Alias [foofoobar] has more than one indices associated with it"));
assertThat(e.getMessage(), containsString("alias [foofoobar] has more than one indices associated with it"));
}
{
@ -894,8 +895,8 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
final String dottedHiddenAlias = ".hidden_alias";
final String dottedHiddenIndex = ".hidden_index";
IndicesOptions excludeHiddenOptions = IndicesOptions.fromOptions(false, false, true, false, false, true, false, false, false);
IndicesOptions includeHiddenOptions = IndicesOptions.fromOptions(false, false, true, false, true, true, false, false, false);
IndicesOptions excludeHiddenOptions = IndicesOptions.fromOptions(false, false, true, false, false, true, false, false, false, true);
IndicesOptions includeHiddenOptions = IndicesOptions.fromOptions(false, false, true, false, true, true, false, false, false, true);
{
// A visible index with a visible alias and a hidden index with a hidden alias
@ -1028,8 +1029,8 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
final String hiddenAlias = "my-hidden-alias";
final String visibleAlias = "my-visible-alias";
IndicesOptions excludeHiddenOptions = IndicesOptions.fromOptions(false, true, true, false, false, true, false, false, false);
IndicesOptions includeHiddenOptions = IndicesOptions.fromOptions(false, true, true, false, true, true, false, false, false);
IndicesOptions excludeHiddenOptions = IndicesOptions.fromOptions(false, true, true, false, false, true, false, false, false, true);
IndicesOptions includeHiddenOptions = IndicesOptions.fromOptions(false, true, true, false, true, true, false, false, false, true);
Metadata.Builder mdBuilder = Metadata.builder()
.put(indexBuilder(hiddenIndex, Settings.builder().put(INDEX_HIDDEN_SETTING.getKey(), true).build())
@ -1710,7 +1711,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
{
Index[] indices = indexNameExpressionResolver.concreteIndices(state,
IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED, "*");
IndicesOptions.STRICT_INCLUDE_DATA_STREAMS_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED, "*");
assertEquals(1, indices.length);
assertEquals("index", indices[0].getName());
}
@ -1722,18 +1723,18 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
}
{
Index[] indices = indexNameExpressionResolver.concreteIndices(state,
IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED, "test-alias");
IndicesOptions.STRICT_INCLUDE_DATA_STREAMS_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED, "test-alias");
assertEquals(0, indices.length);
}
{
Index[] indices = indexNameExpressionResolver.concreteIndices(state,
IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED, "test-*");
IndicesOptions.STRICT_INCLUDE_DATA_STREAMS_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED, "test-*");
assertEquals(1, indices.length);
assertEquals("index", indices[0].getName());
}
{
Index[] indices = indexNameExpressionResolver.concreteIndices(state,
IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED, "ind*", "test-index");
IndicesOptions.STRICT_INCLUDE_DATA_STREAMS_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED, "ind*", "test-index");
assertEquals(1, indices.length);
Arrays.sort(indices, Comparator.comparing(Index::getName));
assertEquals("index", indices[0].getName());
@ -1759,4 +1760,157 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
assertEquals("test-index", indices[2].getName());
}
}
public void testDataStreams() {
final String dataStreamName = "my-data-stream";
IndexMetadata index1 = createBackingIndex(dataStreamName, 1).build();
IndexMetadata index2 = createBackingIndex(dataStreamName, 2).build();
Metadata.Builder mdBuilder = Metadata.builder()
.put(index1, false)
.put(index2, false)
.put(new DataStream(dataStreamName, "ts", org.elasticsearch.common.collect.List.of(index1.getIndex(), index2.getIndex())));
ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
{
IndicesOptions indicesOptions = new IndicesOptions(EnumSet.of(IndicesOptions.Option.INCLUDE_DATA_STREAMS),
EnumSet.of(IndicesOptions.WildcardStates.OPEN));
Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, "my-data-stream");
assertThat(result.length, equalTo(2));
assertThat(result[0].getName(), equalTo(dataStreamName + "-000001"));
assertThat(result[1].getName(), equalTo(dataStreamName + "-000002"));
}
{
// Ignore data streams
IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
Exception e = expectThrows(IllegalArgumentException.class,
() -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, "my-data-stream"));
assertThat(e.getMessage(), equalTo("The provided expression [my-data-stream] matches a " +
"data stream, specify the corresponding concrete indices instead."));
}
{
// Ignore data streams and allow no indices
IndicesOptions indicesOptions = new IndicesOptions(EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES),
EnumSet.of(IndicesOptions.WildcardStates.OPEN));
Exception e = expectThrows(IllegalArgumentException.class,
() -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, "my-data-stream"));
assertThat(e.getMessage(), equalTo("The provided expression [my-data-stream] matches a " +
"data stream, specify the corresponding concrete indices instead."));
}
{
// Ignore data streams, allow no indices and ignore unavailable
IndicesOptions indicesOptions = new IndicesOptions(EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES,
IndicesOptions.Option.IGNORE_UNAVAILABLE), EnumSet.of(IndicesOptions.WildcardStates.OPEN));
Exception e = expectThrows(IllegalArgumentException.class,
() -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, "my-data-stream"));
assertThat(e.getMessage(), equalTo("The provided expression [my-data-stream] matches a " +
"data stream, specify the corresponding concrete indices instead."));
}
{
IndicesOptions indicesOptions = new IndicesOptions(EnumSet.of(IndicesOptions.Option.INCLUDE_DATA_STREAMS),
EnumSet.of(IndicesOptions.WildcardStates.OPEN));
Index result = indexNameExpressionResolver.concreteWriteIndex(state, indicesOptions, "my-data-stream", false);
assertThat(result.getName(), equalTo(dataStreamName + "-000002"));
}
{
// Ignore data streams
IndicesOptions indicesOptions = new IndicesOptions(EnumSet.noneOf(IndicesOptions.Option.class),
EnumSet.of(IndicesOptions.WildcardStates.OPEN));
Exception e = expectThrows(IllegalArgumentException.class,
() -> indexNameExpressionResolver.concreteWriteIndex(state, indicesOptions, "my-data-stream", true));
assertThat(e.getMessage(), equalTo("The provided expression [my-data-stream] matches a " +
"data stream, specify the corresponding concrete indices instead."));
}
{
// Ignore data streams and allow no indices
IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
Exception e = expectThrows(IllegalArgumentException.class,
() -> indexNameExpressionResolver.concreteWriteIndex(state, indicesOptions, "my-data-stream", false));
assertThat(e.getMessage(), equalTo("The provided expression [my-data-stream] matches a data stream, " +
"specify the corresponding concrete indices instead."));
}
{
// Ignore data streams, allow no indices and ignore unavailable
IndicesOptions indicesOptions = new IndicesOptions(EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES,
IndicesOptions.Option.IGNORE_UNAVAILABLE), EnumSet.of(IndicesOptions.WildcardStates.OPEN));
Exception e = expectThrows(IllegalArgumentException.class,
() -> indexNameExpressionResolver.concreteWriteIndex(state, indicesOptions, "my-data-stream", false));
assertThat(e.getMessage(), equalTo("The provided expression [my-data-stream] matches a data stream, " +
"specify the corresponding concrete indices instead."));
}
}
public void testDataStreamsWithWildcardExpression() {
final String dataStream1 = "logs-mysql";
final String dataStream2 = "logs-redis";
IndexMetadata index1 = createBackingIndex(dataStream1, 1).build();
IndexMetadata index2 = createBackingIndex(dataStream1, 2).build();
IndexMetadata index3 = createBackingIndex(dataStream2, 1).build();
IndexMetadata index4 = createBackingIndex(dataStream2, 2).build();
Metadata.Builder mdBuilder = Metadata.builder()
.put(index1, false)
.put(index2, false)
.put(index3, false)
.put(index4, false)
.put(new DataStream(dataStream1, "ts", org.elasticsearch.common.collect.List.of(index1.getIndex(), index2.getIndex())))
.put(new DataStream(dataStream2, "ts", org.elasticsearch.common.collect.List.of(index3.getIndex(), index4.getIndex())));
ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
{
IndicesOptions indicesOptions = new IndicesOptions(EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES,
IndicesOptions.Option.INCLUDE_DATA_STREAMS), EnumSet.of(IndicesOptions.WildcardStates.OPEN));
Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, "logs-*");
Arrays.sort(result, Comparator.comparing(Index::getName));
assertThat(result.length, equalTo(4));
assertThat(result[0].getName(), equalTo(dataStream1 + "-000001"));
assertThat(result[1].getName(), equalTo(dataStream1 + "-000002"));
assertThat(result[2].getName(), equalTo(dataStream2 + "-000001"));
assertThat(result[3].getName(), equalTo(dataStream2 + "-000002"));
}
{
IndicesOptions indicesOptions = new IndicesOptions(EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES,
IndicesOptions.Option.INCLUDE_DATA_STREAMS), EnumSet.of(IndicesOptions.WildcardStates.OPEN));
Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, "logs-m*");
Arrays.sort(result, Comparator.comparing(Index::getName));
assertThat(result.length, equalTo(2));
assertThat(result[0].getName(), equalTo(dataStream1 + "-000001"));
assertThat(result[1].getName(), equalTo(dataStream1 + "-000002"));
}
{
IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN; // without include data streams
Exception e = expectThrows(IllegalArgumentException.class,
() -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, "logs-*"));
assertThat(e.getMessage(), equalTo("The provided expression [logs-*] matches a data stream, " +
"specify the corresponding concrete indices instead."));
}
}
public void testDataStreamsWithRegularIndexAndAlias() {
final String dataStream1 = "logs-foobar";
IndexMetadata index1 = createBackingIndex(dataStream1, 1).build();
IndexMetadata index2 = createBackingIndex(dataStream1, 2).build();
IndexMetadata justAnIndex = IndexMetadata.builder("logs-foobarbaz-0")
.settings(ESTestCase.settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(1)
.putAlias(new AliasMetadata.Builder("logs-foobarbaz"))
.build();
ClusterState state = ClusterState.builder(new ClusterName("_name"))
.metadata(Metadata.builder()
.put(index1, false)
.put(index2, false)
.put(justAnIndex, false)
.put(new DataStream(dataStream1, "ts",
org.elasticsearch.common.collect.List.of(index1.getIndex(), index2.getIndex())))).build();
IndicesOptions indicesOptions = IndicesOptions.strictIncludeDataStreamsExpandOpenAndForbidClosedIgnoreThrottled();
Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, "logs-*");
Arrays.sort(result, Comparator.comparing(Index::getName));
assertThat(result.length, equalTo(3));
assertThat(result[0].getName(), equalTo("logs-foobar-000001"));
assertThat(result[1].getName(), equalTo("logs-foobar-000002"));
assertThat(result[2].getName(), equalTo("logs-foobarbaz-0"));
}
}

101
server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java
View File

@ -45,7 +45,6 @@ import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -53,9 +52,12 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
import static org.elasticsearch.cluster.DataStreamTestHelper.createBackingIndex;
import static org.elasticsearch.cluster.DataStreamTestHelper.createFirstBackingIndex;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.startsWith;
public class MetadataTests extends ESTestCase {
@ -946,49 +948,65 @@ public class MetadataTests extends ESTestCase {
public void testBuilderRejectsDataStreamThatConflictsWithIndex() {
final String dataStreamName = "my-data-stream";
IndexMetadata idx = createFirstBackingIndex(dataStreamName).build();
Metadata.Builder b = Metadata.builder()
.put(idx, false)
.put(IndexMetadata.builder(dataStreamName)
.settings(settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(1)
.build(), false)
.put(new DataStream(dataStreamName, "ts", Collections.emptyList()));
.put(new DataStream(dataStreamName, "ts", org.elasticsearch.common.collect.List.of(idx.getIndex())));
IllegalStateException e = expectThrows(IllegalStateException.class, b::build);
assertThat(e.getMessage(), containsString("data stream [" + dataStreamName + "] conflicts with existing index or alias"));
assertThat(e.getMessage(),
containsString("data stream [" + dataStreamName + "] conflicts with existing concrete index [" + dataStreamName + "]"));
}
public void testBuilderRejectsDataStreamThatConflictsWithAlias() {
final String dataStreamName = "my-data-stream";
IndexMetadata idx = createFirstBackingIndex(dataStreamName + "z")
.putAlias(AliasMetadata.builder(dataStreamName).build())
.build();
Metadata.Builder b = Metadata.builder()
.put(IndexMetadata.builder(dataStreamName + "z")
.settings(settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(1)
.putAlias(AliasMetadata.builder(dataStreamName).build())
.build(), false)
.put(new DataStream(dataStreamName, "ts", Collections.emptyList()));
.put(idx, false)
.put(new DataStream(dataStreamName, "ts", org.elasticsearch.common.collect.List.of(idx.getIndex())));
IllegalStateException e = expectThrows(IllegalStateException.class, b::build);
assertThat(e.getMessage(), containsString("data stream [" + dataStreamName + "] conflicts with existing index or alias"));
assertThat(e.getMessage(),
containsString("data stream [" + dataStreamName + "] conflicts with existing alias [" + dataStreamName + "]"));
}
public void testBuilderRejectsDataStreamWithConflictingBackingIndices() {
final String dataStreamName = "my-data-stream";
final String conflictingIndex = dataStreamName + "-000001";
IndexMetadata validIdx = createFirstBackingIndex(dataStreamName).build();
final String conflictingIndex = dataStreamName + "-000002";
IndexMetadata invalidIdx = createBackingIndex(dataStreamName, 2).build();
Metadata.Builder b = Metadata.builder()
.put(IndexMetadata.builder(conflictingIndex)
.settings(settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(1)
.build(), false)
.put(new DataStream(dataStreamName, "ts", Collections.emptyList()));
.put(validIdx, false)
.put(invalidIdx, false)
.put(new DataStream(dataStreamName, "ts", org.elasticsearch.common.collect.List.of(validIdx.getIndex())));
IllegalStateException e = expectThrows(IllegalStateException.class, b::build);
assertThat(e.getMessage(), containsString("data stream [" + dataStreamName +
"] could create backing indices that conflict with 1 existing index(s) or alias(s) including '" + conflictingIndex + "'"));
}
public void testBuilderRejectsDataStreamWithConflictingBackingAlias() {
final String dataStreamName = "my-data-stream";
final String conflictingName = dataStreamName + "-000002";
IndexMetadata idx = createFirstBackingIndex(dataStreamName)
.putAlias(new AliasMetadata.Builder(conflictingName))
.build();
Metadata.Builder b = Metadata.builder()
.put(idx, false)
.put(new DataStream(dataStreamName, "ts", org.elasticsearch.common.collect.List.of(idx.getIndex())));
IllegalStateException e = expectThrows(IllegalStateException.class, b::build);
assertThat(e.getMessage(), containsString("data stream [" + dataStreamName +
"] could create backing indices that conflict with 1 existing index(s) or alias(s) including '" + conflictingName + "'"));
}
public void testBuilderForDataStreamWithRandomlyNumberedBackingIndices() {
final String dataStreamName = "my-data-stream";
final List<Index> backingIndices = new ArrayList<>();
@ -1012,6 +1030,37 @@ public class MetadataTests extends ESTestCase {
assertThat(metadata.dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName));
}
public void testBuildIndicesLookupForDataStreams() {
Metadata.Builder b = Metadata.builder();
int numDataStreams = randomIntBetween(2, 8);
for (int i = 0; i < numDataStreams; i++) {
String name = "data-stream-" + i;
int numBackingIndices = randomIntBetween(1, 4);
List<Index> indices = new ArrayList<>(numBackingIndices);
for (int j = 1; j <= numBackingIndices; j++) {
IndexMetadata idx = createBackingIndex(name, j).build();
indices.add(idx.getIndex());
b.put(idx, true);
}
b.put(new DataStream(name, "ts", indices));
}
Metadata metadata = b.build();
assertThat(metadata.dataStreams().size(), equalTo(numDataStreams));
for (int i = 0; i < numDataStreams; i++) {
String name = "data-stream-" + i;
IndexAbstraction value = metadata.getIndicesLookup().get(name);
assertThat(value, notNullValue());
DataStream ds = metadata.dataStreams().get(name);
assertThat(ds, notNullValue());
assertThat(value.isHidden(), is(false));
assertThat(value.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM));
assertThat(value.getIndices().size(), equalTo(ds.getIndices().size()));
assertThat(value.getWriteIndex().getIndex().getName(), equalTo(name + "-00000" + ds.getIndices().size()));
}
}
public void testSerialization() throws IOException {
final Metadata orig = randomMetadata();
final BytesStreamOutput out = new BytesStreamOutput();
@ -1023,7 +1072,9 @@ public class MetadataTests extends ESTestCase {
}
public static Metadata randomMetadata() {
return Metadata.builder()
DataStream randomDataStream = DataStreamTests.randomInstance();
Metadata.Builder md = Metadata.builder()
.put(buildIndexMetadata("index", "alias", randomBoolean() ? null : randomBoolean()).build(), randomBoolean())
.put(IndexTemplateMetadata.builder("template" + randomAlphaOfLength(3))
.patterns(Arrays.asList("bar-*", "foo-*"))
@ -1043,7 +1094,15 @@ public class MetadataTests extends ESTestCase {
.version(randomNonNegativeLong())
.put("component_template_" + randomAlphaOfLength(3), ComponentTemplateTests.randomInstance())
.put("index_template_v2_" + randomAlphaOfLength(3), IndexTemplateV2Tests.randomInstance())
.put(DataStreamTests.randomInstance())
.build();
.put(randomDataStream);
for (Index index : randomDataStream.getIndices()) {
md.put(IndexMetadata.builder(index.getName())
.settings(ESTestCase.settings(Version.CURRENT).put("index.hidden", true))
.numberOfShards(1)
.numberOfReplicas(1));
}
return md.build();
}
}

14
server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java
View File

@ -40,16 +40,20 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import static org.elasticsearch.cluster.DataStreamTestHelper.createFirstBackingIndex;
import static org.elasticsearch.cluster.metadata.AliasMetadata.newAliasMetadataBuilder;
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED;
import static org.elasticsearch.cluster.metadata.Metadata.CONTEXT_MODE_API;
import static org.elasticsearch.cluster.metadata.Metadata.CONTEXT_MODE_GATEWAY;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
public class ToAndFromJsonMetadataTests extends ESTestCase {
public void testSimpleJsonFromAndTo() throws IOException {
IndexMetadata idx1 = createFirstBackingIndex("data-stream1").build();
IndexMetadata idx2 = createFirstBackingIndex("data-stream2").build();
Metadata metadata = Metadata.builder()
.put(IndexTemplateMetadata.builder("foo")
.patterns(Collections.singletonList("bar"))
@ -94,8 +98,10 @@ public class ToAndFromJsonMetadataTests extends ESTestCase {
.putAlias(newAliasMetadataBuilder("alias-bar1"))
.putAlias(newAliasMetadataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}"))
.putAlias(newAliasMetadataBuilder("alias-bar3").routing("routing-bar")))
.put(new DataStream("data-stream1", "@timestamp", Collections.emptyList()))
.put(new DataStream("data-stream2", "@timestamp2", Collections.emptyList()))
.put(idx1, false)
.put(idx2, false)
.put(new DataStream("data-stream1", "@timestamp", org.elasticsearch.common.collect.List.of(idx1.getIndex())))
.put(new DataStream("data-stream2", "@timestamp2", org.elasticsearch.common.collect.List.of(idx2.getIndex())))
.build();
XContentBuilder builder = JsonXContent.contentBuilder();
@ -146,11 +152,11 @@ public class ToAndFromJsonMetadataTests extends ESTestCase {
assertNotNull(parsedMetadata.dataStreams().get("data-stream1"));
assertThat(parsedMetadata.dataStreams().get("data-stream1").getName(), is("data-stream1"));
assertThat(parsedMetadata.dataStreams().get("data-stream1").getTimeStampField(), is("@timestamp"));
assertThat(parsedMetadata.dataStreams().get("data-stream1").getIndices(), is(Collections.emptyList()));
assertThat(parsedMetadata.dataStreams().get("data-stream1").getIndices(), contains(idx1.getIndex()));
assertNotNull(parsedMetadata.dataStreams().get("data-stream2"));
assertThat(parsedMetadata.dataStreams().get("data-stream2").getName(), is("data-stream2"));
assertThat(parsedMetadata.dataStreams().get("data-stream2").getTimeStampField(), is("@timestamp2"));
assertThat(parsedMetadata.dataStreams().get("data-stream2").getIndices(), is(Collections.emptyList()));
assertThat(parsedMetadata.dataStreams().get("data-stream2").getIndices(), contains(idx2.getIndex()));
}
private static final String MAPPING_SOURCE1 = "{\"mapping1\":{\"text1\":{\"type\":\"string\"}}}";

182
server/src/test/java/org/elasticsearch/indices/DataStreamIT.java Normal file
View File

@ -0,0 +1,182 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.indices;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction;
import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction;
import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.test.ESIntegTestCase;
import java.util.Arrays;
import java.util.Comparator;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
public class DataStreamIT extends ESIntegTestCase {
public void testBasicScenario() throws Exception {
CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-foo");
createDataStreamRequest.setTimestampFieldName("@timestamp1");
client().admin().indices().createDataStream(createDataStreamRequest).get();
createDataStreamRequest = new CreateDataStreamAction.Request("metrics-bar");
createDataStreamRequest.setTimestampFieldName("@timestamp2");
client().admin().indices().createDataStream(createDataStreamRequest).get();
GetDataStreamsAction.Request getDataStreamRequest = new GetDataStreamsAction.Request("*");
GetDataStreamsAction.Response getDataStreamResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet();
getDataStreamResponse.getDataStreams().sort(Comparator.comparing(DataStream::getName));
assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(2));
assertThat(getDataStreamResponse.getDataStreams().get(0).getName(), equalTo("metrics-bar"));
assertThat(getDataStreamResponse.getDataStreams().get(0).getTimeStampField(), equalTo("@timestamp2"));
assertThat(getDataStreamResponse.getDataStreams().get(0).getIndices().size(), equalTo(1));
assertThat(getDataStreamResponse.getDataStreams().get(0).getIndices().get(0).getName(), equalTo("metrics-bar-000001"));
assertThat(getDataStreamResponse.getDataStreams().get(1).getName(), equalTo("metrics-foo"));
assertThat(getDataStreamResponse.getDataStreams().get(1).getTimeStampField(), equalTo("@timestamp1"));
assertThat(getDataStreamResponse.getDataStreams().get(1).getIndices().size(), equalTo(1));
assertThat(getDataStreamResponse.getDataStreams().get(1).getIndices().get(0).getName(), equalTo("metrics-foo-000001"));
GetIndexResponse getIndexResponse =
client().admin().indices().getIndex(new GetIndexRequest().indices("metrics-bar-000001")).actionGet();
assertThat(getIndexResponse.getSettings().get("metrics-bar-000001"), notNullValue());
assertThat(getIndexResponse.getSettings().get("metrics-bar-000001").getAsBoolean("index.hidden", null), is(true));
getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices("metrics-foo-000001")).actionGet();
assertThat(getIndexResponse.getSettings().get("metrics-foo-000001"), notNullValue());
assertThat(getIndexResponse.getSettings().get("metrics-foo-000001").getAsBoolean("index.hidden", null), is(true));
int numDocsBar = randomIntBetween(2, 16);
indexDocs("metrics-bar", numDocsBar);
int numDocsFoo = randomIntBetween(2, 16);
indexDocs("metrics-foo", numDocsFoo);
verifyDocs("metrics-bar", numDocsBar);
verifyDocs("metrics-foo", numDocsFoo);
// TODO: execute rollover and index some more data.
DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request("metrics-*");
client().admin().indices().deleteDataStream(deleteDataStreamRequest).actionGet();
getDataStreamResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet();
assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(0));
expectThrows(IndexNotFoundException.class,
() -> client().admin().indices().getIndex(new GetIndexRequest().indices("metrics-bar-000001")).actionGet());
expectThrows(IndexNotFoundException.class,
() -> client().admin().indices().getIndex(new GetIndexRequest().indices("metrics-foo-000001")).actionGet());
}
public void testOtherWriteOps() throws Exception {
String dataStreamName = "metrics-foobar";
CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
createDataStreamRequest.setTimestampFieldName("@timestamp1");
client().admin().indices().createDataStream(createDataStreamRequest).get();
{
BulkRequest bulkRequest = new BulkRequest()
.add(new IndexRequest(dataStreamName).source("{}", XContentType.JSON));
expectFailure(dataStreamName, () -> client().bulk(bulkRequest).actionGet());
}
{
BulkRequest bulkRequest = new BulkRequest()
.add(new DeleteRequest(dataStreamName, "_id"));
expectFailure(dataStreamName, () -> client().bulk(bulkRequest).actionGet());
}
{
BulkRequest bulkRequest = new BulkRequest()
.add(new UpdateRequest(dataStreamName, "_id").doc("{}", XContentType.JSON));
expectFailure(dataStreamName, () -> client().bulk(bulkRequest).actionGet());
}
{
IndexRequest indexRequest = new IndexRequest(dataStreamName).source("{}", XContentType.JSON);
expectFailure(dataStreamName, () -> client().index(indexRequest).actionGet());
}
{
UpdateRequest updateRequest = new UpdateRequest(dataStreamName, "_id")
.doc("{}", XContentType.JSON);
expectFailure(dataStreamName, () -> client().update(updateRequest).actionGet());
}
{
DeleteRequest deleteRequest = new DeleteRequest(dataStreamName, "_id");
expectFailure(dataStreamName, () -> client().delete(deleteRequest).actionGet());
}
{
IndexRequest indexRequest = new IndexRequest(dataStreamName).source("{}", XContentType.JSON)
.opType(DocWriteRequest.OpType.CREATE);
IndexResponse indexResponse = client().index(indexRequest).actionGet();
assertThat(indexResponse.getIndex(), equalTo(dataStreamName + "-000001"));
}
{
BulkRequest bulkRequest = new BulkRequest()
.add(new IndexRequest(dataStreamName).source("{}", XContentType.JSON)
.opType(DocWriteRequest.OpType.CREATE));
BulkResponse bulkItemResponses = client().bulk(bulkRequest).actionGet();
assertThat(bulkItemResponses.getItems()[0].getIndex(), equalTo(dataStreamName + "-000001"));
}
DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request("*");
client().admin().indices().deleteDataStream(deleteDataStreamRequest).actionGet();
}
private static void indexDocs(String dataStream, int numDocs) {
BulkRequest bulkRequest = new BulkRequest();
for (int i = 0; i < numDocs; i++) {
bulkRequest.add(new IndexRequest(dataStream)
.opType(DocWriteRequest.OpType.CREATE)
.source("{}", XContentType.JSON));
}
client().bulk(bulkRequest).actionGet();
client().admin().indices().refresh(new RefreshRequest(dataStream)).actionGet();
}
private static void verifyDocs(String dataStream, long expectedNumHits) {
SearchRequest searchRequest = new SearchRequest(dataStream);
searchRequest.source().size((int) expectedNumHits);
SearchResponse searchResponse = client().search(searchRequest).actionGet();
assertThat(searchResponse.getHits().getTotalHits().value, equalTo(expectedNumHits));
Arrays.stream(searchResponse.getHits().getHits()).forEach(hit -> {
assertThat(hit.getIndex(), equalTo(dataStream + "-000001"));
});
}
private static void expectFailure(String dataStreamName, ThrowingRunnable runnable) {
Exception e = expectThrows(IllegalArgumentException.class, runnable);
assertThat(e.getMessage(), equalTo("The provided expression [" + dataStreamName +
"] matches a data stream, specify the corresponding concrete indices instead."));
}
}

91
server/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
View File

@ -19,11 +19,14 @@
package org.elasticsearch.indices;
import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequestBuilder;
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequestBuilder;
import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistRequestBuilder;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder;
import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder;
import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction;
import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction;
import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequestBuilder;
import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder;
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder;
@ -35,6 +38,7 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequestBui
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder;
import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
import org.elasticsearch.action.search.MultiSearchRequestBuilder;
import org.elasticsearch.action.search.MultiSearchResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
@ -45,6 +49,7 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
@ -60,6 +65,7 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
@ -642,6 +648,91 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
verify(client().admin().indices().prepareUpdateSettings("baz*").setSettings(Settings.builder().put("a", "b")), true);
}
public void testDataStreamsResolvability() {
String dataStreamName = "logs-foobar";
CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(dataStreamName);
request.setTimestampFieldName("ts");
client().admin().indices().createDataStream(request).actionGet();
verifyResolvability(dataStreamName, client().prepareIndex(dataStreamName, "_doc")
.setSource("{}", XContentType.JSON)
.setOpType(DocWriteRequest.OpType.CREATE),
false);
verifyResolvability(dataStreamName, refreshBuilder(dataStreamName), false);
verifyResolvability(dataStreamName, search(dataStreamName), false, 1);
verifyResolvability(dataStreamName, msearch(null, dataStreamName), false);
verifyResolvability(dataStreamName, clearCache(dataStreamName), true);
verifyResolvability(dataStreamName, _flush(dataStreamName),true);
verifyResolvability(dataStreamName, segments(dataStreamName), true);
verifyResolvability(dataStreamName, stats(dataStreamName), true);
verifyResolvability(dataStreamName, forceMerge(dataStreamName), true);
verifyResolvability(dataStreamName, validateQuery(dataStreamName), true);
verifyResolvability(dataStreamName, getAliases(dataStreamName), true);
verifyResolvability(dataStreamName, getFieldMapping(dataStreamName), true);
verifyResolvability(dataStreamName, getMapping(dataStreamName), true);
verifyResolvability(dataStreamName, getSettings(dataStreamName), true);
request = new CreateDataStreamAction.Request("logs-barbaz");
request.setTimestampFieldName("ts");
client().admin().indices().createDataStream(request).actionGet();
verifyResolvability("logs-barbaz", client().prepareIndex("logs-barbaz", "_doc")
.setSource("{}", XContentType.JSON)
.setOpType(DocWriteRequest.OpType.CREATE),
false);
String wildcardExpression = "logs*";
verifyResolvability(wildcardExpression, refreshBuilder(wildcardExpression), false);
verifyResolvability(wildcardExpression, search(wildcardExpression), false, 2);
verifyResolvability(wildcardExpression, msearch(null, wildcardExpression), false);
verifyResolvability(wildcardExpression, clearCache(wildcardExpression), true);
verifyResolvability(wildcardExpression, _flush(wildcardExpression),true);
verifyResolvability(wildcardExpression, segments(wildcardExpression), true);
verifyResolvability(wildcardExpression, stats(wildcardExpression), true);
verifyResolvability(wildcardExpression, forceMerge(wildcardExpression), true);
verifyResolvability(wildcardExpression, validateQuery(wildcardExpression), true);
verifyResolvability(wildcardExpression, getAliases(wildcardExpression), true);
verifyResolvability(wildcardExpression, getFieldMapping(wildcardExpression), true);
verifyResolvability(wildcardExpression, getMapping(wildcardExpression), true);
verifyResolvability(wildcardExpression, getSettings(wildcardExpression), true);
DeleteDataStreamAction.Request deleteRequest = new DeleteDataStreamAction.Request("*");
client().admin().indices().deleteDataStream(deleteRequest).actionGet();
}
private static void verifyResolvability(String dataStream, ActionRequestBuilder requestBuilder, boolean fail) {
verifyResolvability(dataStream, requestBuilder, fail, 0);
}
private static void verifyResolvability(String dataStream, ActionRequestBuilder requestBuilder, boolean fail, long expectedCount) {
if (fail) {
String expectedErrorMessage = "The provided expression [" + dataStream +
"] matches a data stream, specify the corresponding concrete indices instead.";
if (requestBuilder instanceof MultiSearchRequestBuilder) {
MultiSearchResponse multiSearchResponse = ((MultiSearchRequestBuilder) requestBuilder).get();
assertThat(multiSearchResponse.getResponses().length, equalTo(1));
assertThat(multiSearchResponse.getResponses()[0].isFailure(), is(true));
assertThat(multiSearchResponse.getResponses()[0].getFailure(), instanceOf(IllegalArgumentException.class));
assertThat(multiSearchResponse.getResponses()[0].getFailure().getMessage(), equalTo(expectedErrorMessage));
} else if (requestBuilder instanceof ValidateQueryRequestBuilder) {
ValidateQueryResponse response = (ValidateQueryResponse) requestBuilder.get();
assertThat(response.getQueryExplanation().get(0).getError(), equalTo(expectedErrorMessage));
} else {
Exception e = expectThrows(IllegalArgumentException.class, requestBuilder::get);
assertThat(e.getMessage(), equalTo(expectedErrorMessage));
}
} else {
if (requestBuilder instanceof SearchRequestBuilder) {
SearchRequestBuilder searchRequestBuilder = (SearchRequestBuilder) requestBuilder;
assertHitCount(searchRequestBuilder.get(), expectedCount);
} else if (requestBuilder instanceof MultiSearchRequestBuilder) {
MultiSearchResponse multiSearchResponse = ((MultiSearchRequestBuilder) requestBuilder).get();
assertThat(multiSearchResponse.getResponses()[0].isFailure(), is(false));
} else {
requestBuilder.get();
}
}
}
private static SearchRequestBuilder search(String... indices) {
return client().prepareSearch(indices).setQuery(matchAllQuery());
}

40
test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java Normal file
View File

@ -0,0 +1,40 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.cluster;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.test.ESTestCase;
import java.util.Locale;
public final class DataStreamTestHelper {
public static IndexMetadata.Builder createFirstBackingIndex(String dataStreamName) {
return createBackingIndex(dataStreamName, 1);
}
public static IndexMetadata.Builder createBackingIndex(String dataStreamName, int generation) {
return IndexMetadata.builder(String.format(Locale.ROOT, "%s-%06d", dataStreamName, generation))
.settings(ESTestCase.settings(Version.CURRENT).put("index.hidden", true))
.numberOfShards(1)
.numberOfReplicas(1);
}
}

2
test/framework/src/main/java/org/elasticsearch/test/TestCluster.java
View File

@ -142,7 +142,7 @@ public abstract class TestCluster implements Closeable {
try {
// include wiping hidden indices!
assertAcked(client().admin().indices().prepareDelete(indices)
.setIndicesOptions(IndicesOptions.fromOptions(false, true, true, true, true, false, false, true, false)));
.setIndicesOptions(IndicesOptions.fromOptions(false, true, true, true, true, false, false, true, false, false)));
} catch (IndexNotFoundException e) {
// ignore
} catch (IllegalArgumentException e) {

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidator.java
View File

@ -228,7 +228,7 @@ public final class SourceDestValidator {
// note: this is equivalent to the default for search requests
private static final IndicesOptions DEFAULT_INDICES_OPTIONS_FOR_VALIDATION = IndicesOptions
.strictExpandOpenAndForbidClosedIgnoreThrottled();
.strictIncludeDataStreamsExpandOpenAndForbidClosedIgnoreThrottled();
public static final SourceDestValidation SOURCE_MISSING_VALIDATION = new SourceMissingValidation();
public static final SourceDestValidation REMOTE_SOURCE_VALIDATION = new RemoteSourceEnabledAndRemoteLicenseValidation();

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotHistoryStore.java
View File

@ -133,7 +133,7 @@ public class SnapshotHistoryStore {
} else if (slmHistory.getType() != IndexAbstraction.Type.ALIAS) {
// This is not an alias, error out
andThen.onFailure(new IllegalStateException("SLM history alias [" + SLM_HISTORY_ALIAS +
"] already exists as concrete index"));
"] already exists as " + slmHistory.getType().getDisplayName()));
} else {
logger.error("unexpected IndexOrAlias for [{}]: [{}]", SLM_HISTORY_ALIAS, slmHistory);
// (slmHistory.isAlias() == true) but (slmHistory instanceof Alias == false)?

2
x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java
View File

@ -221,7 +221,7 @@ public class ILMHistoryStore implements Closeable {
} else if (ilmHistory.getType() != IndexAbstraction.Type.ALIAS) {
// This is not an alias, error out
listener.onFailure(new IllegalStateException("ILM history alias [" + ILM_HISTORY_ALIAS +
"] already exists as concrete index"));
"] already exists as " + ilmHistory.getType().getDisplayName()));
} else {
logger.error("unexpected IndexOrAlias for [{}]: [{}]", ILM_HISTORY_ALIAS, ilmHistory);
assert false : ILM_HISTORY_ALIAS + " cannot be both an alias and not an alias simultaneously";

10
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java
View File

@ -268,7 +268,8 @@ public class DatafeedNodeSelectorTests extends ESTestCase {
equalTo("cannot start datafeed [datafeed_id] because it failed resolving indices given [not_foo] and " +
"indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, " +
"expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, " +
"forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true]] with exception [no such index [not_foo]]"));
"forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true, include_data_streams=true]] " +
"with exception [no such index [not_foo]]"));
ElasticsearchException e = expectThrows(ElasticsearchException.class,
() -> new DatafeedNodeSelector(clusterState,
@ -281,8 +282,8 @@ public class DatafeedNodeSelectorTests extends ESTestCase {
+ "[cannot start datafeed [datafeed_id] because it failed resolving " +
"indices given [not_foo] and indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, " +
"expand_wildcards_open=true, expand_wildcards_closed=false, expand_wildcards_hidden=false, " +
"allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true]] " +
"with exception [no such index [not_foo]]]"));
"allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true, " +
"include_data_streams=true]] with exception [no such index [not_foo]]]"));
}
public void testRemoteIndex() {
@ -380,7 +381,8 @@ public class DatafeedNodeSelectorTests extends ESTestCase {
+ "[cannot start datafeed [datafeed_id] because it failed resolving indices given [not_foo] and " +
"indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, " +
"expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, " +
"forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true]] with exception [no such index [not_foo]]]"));
"forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true, include_data_streams=true]] " +
"with exception [no such index [not_foo]]]"));
}
public void testSelectNode_GivenMlUpgradeMode() {

8
x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
View File

@ -438,6 +438,14 @@ class IndicesAndAliasesResolver {
return false;
}
}
if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) {
// If indicesOptions.includeDataStreams() returns false then we fail later in IndexNameExpressionResolver.
if (isHidden == false || indicesOptions.expandWildcardsHidden()) {
return true;
} else {
return false;
}
}
assert indexAbstraction.getIndices().size() == 1 : "concrete index must point to a single index";
IndexMetadata indexMetadata = indexAbstraction.getIndices().get(0);
if (isHidden && indicesOptions.expandWildcardsHidden() == false && isVisibleDueToImplicitHidden(expression, index) == false) {

72
x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
View File

@ -33,7 +33,9 @@ import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
import org.elasticsearch.cluster.DataStreamTestHelper;
import org.elasticsearch.cluster.metadata.AliasMetadata;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexMetadata.State;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@ -121,6 +123,11 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
final boolean withAlias = randomBoolean();
final String securityIndexName = SECURITY_MAIN_ALIAS + (withAlias ? "-" + randomAlphaOfLength(5) : "");
final String dataStreamName = "logs-foobar";
final String otherDataStreamName = "logs-foo";
IndexMetadata dataStreamIndex1 = DataStreamTestHelper.createBackingIndex(dataStreamName, 1).build();
IndexMetadata dataStreamIndex2 = DataStreamTestHelper.createBackingIndex(dataStreamName, 2).build();
IndexMetadata dataStreamIndex3 = DataStreamTestHelper.createBackingIndex(otherDataStreamName, 1).build();
Metadata metadata = Metadata.builder()
.put(indexBuilder("foo").putAlias(AliasMetadata.builder("foofoobar"))
.putAlias(AliasMetadata.builder("foounauthorized")).settings(settings))
@ -157,6 +164,13 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
.put(indexBuilder("visible-w-aliases").settings(Settings.builder().put(settings).build())
.putAlias(AliasMetadata.builder("alias-visible").build())
.putAlias(AliasMetadata.builder("alias-visible-mixed").isHidden(false).build()))
.put(dataStreamIndex1, true)
.put(dataStreamIndex2, true)
.put(dataStreamIndex3, true)
.put(new DataStream(dataStreamName, "ts",
org.elasticsearch.common.collect.List.of(dataStreamIndex1.getIndex(), dataStreamIndex2.getIndex())))
.put(new DataStream(otherDataStreamName, "ts",
org.elasticsearch.common.collect.List.of(dataStreamIndex3.getIndex())))
.put(indexBuilder(securityIndexName).settings(settings)).build();
if (withAlias) {
@ -188,6 +202,20 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
.build()
}, null));
roleMap.put(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName(), ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR);
roleMap.put("data_stream_test1", new RoleDescriptor("data_stream_test1", null,
new IndicesPrivileges[] {
IndicesPrivileges.builder()
.indices(dataStreamName + "*")
.privileges("all")
.build()
}, null));
roleMap.put("data_stream_test2", new RoleDescriptor("data_stream_test2", null,
new IndicesPrivileges[] {
IndicesPrivileges.builder()
.indices(otherDataStreamName + "*")
.privileges("all")
.build()
}, null));
final FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
doAnswer((i) -> {
ActionListener callback =
@ -1452,7 +1480,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
// closed + hidden, ignore aliases
searchRequest = new SearchRequest();
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, false, true, true, true, false, true, false));
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, false, true, true, true, false, true, false, true));
authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), containsInAnyOrder("bar-closed", "foofoo-closed", "hidden-closed", ".hidden-closed"));
@ -1468,7 +1496,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
// allow no indices, do not expand to open or closed, expand hidden, ignore aliases
searchRequest = new SearchRequest();
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, true, false, false, false, true, false, true, false));
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, true, false, false, false, true, false, true, false, true));
authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), contains("-*"));
@ -1510,19 +1538,55 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
// Make sure ignoring aliases works (visible only)
searchRequest = new SearchRequest();
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, true, true, false, false, true, false, true, false));
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, true, true, false, false, true, false, true, false, true));
resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), contains("-*"));
assertThat(resolvedIndices.getRemote(), emptyIterable());
// Make sure ignoring aliases works (including hidden)
searchRequest = new SearchRequest();
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, true, true, false, true, false));
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, true, true, false, true, false, true));
resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), containsInAnyOrder("hidden-open"));
assertThat(resolvedIndices.getRemote(), emptyIterable());
}
public void testDataStreamResolution() {
{
final User user = new User("data-steam-tester1", "data_stream_test1");
final List<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
// Resolve data streams:
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("logs-*");
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, false, true, true, true, true, true));
ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), contains("logs-foobar"));
assertThat(resolvedIndices.getRemote(), emptyIterable());
// Ignore data streams:
searchRequest = new SearchRequest();
searchRequest.indices("logs-*");
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, true, true, false, false, true, true, true, true, false));
resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
// if data streams are to be ignored then this happens in IndexNameExpressionResolver:
assertThat(resolvedIndices.getLocal(), contains("logs-foobar"));
assertThat(resolvedIndices.getRemote(), emptyIterable());
}
{
final User user = new User("data-steam-tester2", "data_stream_test2");
final List<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
// Resolve *all* data streams:
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("logs-*");
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, false, true, true, true, true, true));
ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), containsInAnyOrder("logs-foo", "logs-foobar"));
assertThat(resolvedIndices.getRemote(), emptyIterable());
}
}
private List<String> buildAuthorizedIndices(User user, String action) {
PlainActionFuture<Role> rolesListener = new PlainActionFuture<>();
final Authentication authentication =