diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index e2e9af835c2..2486cc47ffa 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -2039,7 +2039,8 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase { AliasMetadata alias = AliasMetadata.builder("alias").writeIndex(true).build(); Template template = new Template(settings, mappings, Collections.singletonMap("alias", alias)); List pattern = Collections.singletonList("pattern"); - IndexTemplateV2 indexTemplate = new IndexTemplateV2(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>()); + IndexTemplateV2 indexTemplate = + new IndexTemplateV2(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null); PutIndexTemplateV2Request putIndexTemplateV2Request = new PutIndexTemplateV2Request().name(templateName).create(true).indexTemplate(indexTemplate); @@ -2085,7 +2086,8 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase { AliasMetadata alias = AliasMetadata.builder("alias").writeIndex(true).build(); Template template = new Template(settings, mappings, org.elasticsearch.common.collect.Map.of("alias", alias)); List pattern = org.elasticsearch.common.collect.List.of("pattern"); - IndexTemplateV2 indexTemplate = new IndexTemplateV2(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>()); + IndexTemplateV2 indexTemplate = + new IndexTemplateV2(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null); PutIndexTemplateV2Request putIndexTemplateV2Request = new PutIndexTemplateV2Request().name(templateName).create(true).indexTemplate(indexTemplate); @@ -2097,7 +2099,7 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase { AliasMetadata simulationAlias = AliasMetadata.builder("simulation-alias").writeIndex(true).build(); IndexTemplateV2 simulationTemplate = new IndexTemplateV2(pattern, new Template(null, null, org.elasticsearch.common.collect.Map.of("simulation-alias", simulationAlias)), Collections.emptyList(), 2L, 1L, - new HashMap<>()); + new HashMap<>(), null); PutIndexTemplateV2Request newIndexTemplateReq = new PutIndexTemplateV2Request().name("used-for-simulation").create(true).indexTemplate(indexTemplate); newIndexTemplateReq.indexTemplate(simulationTemplate); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesV2ResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesV2ResponseTests.java index f5977cf6901..f403c47d9f2 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesV2ResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesV2ResponseTests.java @@ -75,6 +75,7 @@ public class GetIndexTemplatesV2ResponseTests extends ESTestCase { List patterns = Arrays.asList(generateRandomStringArray(10, 10, false, false)); List composedOf = null; Map meta = null; + IndexTemplateV2.DataStreamTemplate dataStreamTemplate = null; if (randomBoolean()) { composedOf = Arrays.asList(generateRandomStringArray(10, 10, false, false)); } @@ -84,6 +85,9 @@ public class GetIndexTemplatesV2ResponseTests extends ESTestCase { Long priority = randomBoolean() ? null : randomNonNegativeLong(); Long version = randomBoolean() ? null : randomNonNegativeLong(); - return new IndexTemplateV2(patterns, randomTemplate(), composedOf, priority, version, meta); + if (randomBoolean()) { + dataStreamTemplate = new IndexTemplateV2.DataStreamTemplate(randomAlphaOfLength(8)); + } + return new IndexTemplateV2(patterns, randomTemplate(), composedOf, priority, version, meta, dataStreamTemplate); } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml index 4c61912fdf0..46a98d3d7f1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml @@ -1,8 +1,8 @@ --- "Create data stream": - skip: - version: " - 7.7.99" - reason: available only in 7.8+ + version: " - 7.8.99" + reason: "data streams only supported in 7.9+" - do: indices.create_data_stream: @@ -62,8 +62,8 @@ --- "Create data stream with invalid name": - skip: - version: " - 7.7.99" - reason: available only in 7.8+ + version: " - 7.8.99" + reason: "data streams only supported in 7.9+" - do: catch: bad_request @@ -78,8 +78,8 @@ --- "Get data stream": - skip: - version: " - 7.7.99" - reason: available only in 7.8+ + version: " - 7.8.99" + reason: "data streams only supported in 7.9+" - do: indices.create_data_stream: @@ -147,8 +147,8 @@ --- "Delete data stream with backing indices": - skip: - version: " - 7.7.99" - reason: available only in 7.8+ + version: " - 7.8.99" + reason: data streams only supported in 7.9+ - do: indices.create_data_stream: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml index daff3aafccc..132dd8f4c47 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml @@ -1,8 +1,8 @@ --- "Test apis that do not supported data streams": - skip: - version: " - 7.7.99" - reason: available only in 7.8+ + version: " - 7.8.99" + reason: "data streams only supported in 7.9+" - do: indices.create_data_stream: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/30_auto_create_data_stream.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/30_auto_create_data_stream.yml new file mode 100644 index 00000000000..bd0e11cc85f --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/30_auto_create_data_stream.yml @@ -0,0 +1,51 @@ +--- +"Put index template": + - skip: + version: " - 7.8.99" + reason: "data streams supported from 7.9" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation" + indices.put_index_template: + name: generic_logs_template + body: + index_patterns: logs-* + data_stream: + timestamp_field: timestamp + template: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + timestamp: + type: date + + - do: + index: + index: logs-foobar + refresh: true + body: { foo: bar } + + - do: + search: + index: logs-foobar + body: { query: { match_all: {} } } + - length: { hits.hits: 1 } + - match: { hits.hits.0._index: logs-foobar-000001 } + - match: { hits.hits.0._source.foo: 'bar' } + + - do: + indices.get_data_streams: + name: logs-foobar + - match: { 0.name: logs-foobar } + - match: { 0.timestamp_field: 'timestamp' } + - length: { 0.indices: 1 } + - match: { 0.indices.0.index_name: 'logs-foobar-000001' } + + - do: + indices.delete_data_stream: + name: logs-foobar + - is_true: acknowledged diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml index aa6126e29e1..b6d30fc7cd5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml @@ -1,8 +1,8 @@ --- "Delete backing index on data stream": - skip: - version: " - 7.99.99" - reason: "enable in 7.8+ after backporting" + version: " - 7.8.99" + reason: "data streams only supported in 7.9+" - do: indices.create_data_stream: @@ -55,8 +55,8 @@ --- "Attempt to delete write index on data stream is rejected": - skip: - version: " - 7.99.99" - reason: "enable in 7.8+ after backporting" + version: " - 7.8.99" + reason: "data streams only supported in 7.9+" - do: indices.create_data_stream: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml index 8408e68cd8d..9e5320e844a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml @@ -1,8 +1,8 @@ --- "Get backing indices for data stream": - skip: - version: " - 7.99.99" - reason: "enable in 7.8+ after backporting" + version: " - 7.8.99" + reason: "data streams only supported in 7.9+" - do: indices.create_data_stream: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml index c2bc48642d6..6cf1ded5af7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml @@ -1,8 +1,8 @@ --- "Roll over a data stream": - skip: - version: " - 7.99.99" - reason: "enable in 7.8+ after backporting" + version: " - 7.8.99" + reason: "data streams only supported in 7.9+" - do: indices.create_data_stream: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java index 846e1847f15..7052121e128 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java @@ -24,14 +24,25 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateV2Action; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateV2Action; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.replication.ReplicationRequest; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexTemplateV2; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.ingest.IngestTestPlugin; @@ -44,20 +55,25 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import static org.elasticsearch.action.DocWriteRequest.OpType.CREATE; import static org.elasticsearch.action.DocWriteResponse.Result.CREATED; import static org.elasticsearch.action.DocWriteResponse.Result.UPDATED; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasItemInArray; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.oneOf; @@ -202,4 +218,87 @@ public class BulkIntegrationIT extends ESIntegTestCase { assertFalse(thread.isAlive()); } } + + public void testMixedAutoCreate() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build(); + + PutIndexTemplateV2Action.Request createTemplateRequest = new PutIndexTemplateV2Action.Request("logs-foo"); + createTemplateRequest.indexTemplate( + new IndexTemplateV2( + Collections.singletonList("logs-foo*"), + new Template(settings, null, null), + null, null, null, null, + new IndexTemplateV2.DataStreamTemplate("@timestamp")) + ); + client().execute(PutIndexTemplateV2Action.INSTANCE, createTemplateRequest).actionGet(); + + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-foobaz").opType(CREATE).source("{}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-barfoo").opType(CREATE).source("{}", XContentType.JSON)); + BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); + assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); + + bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-foobaz2").opType(CREATE).source("{}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-barfoo2").opType(CREATE).source("{}", XContentType.JSON)); + bulkResponse = client().bulk(bulkRequest).actionGet(); + assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); + + bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-foobaz2").opType(CREATE).source("{}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-foobaz3").opType(CREATE).source("{}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-barfoo2").opType(CREATE).source("{}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-barfoo3").opType(CREATE).source("{}", XContentType.JSON)); + bulkResponse = client().bulk(bulkRequest).actionGet(); + assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); + + GetDataStreamsAction.Request getDataStreamRequest = new GetDataStreamsAction.Request("*"); + GetDataStreamsAction.Response getDataStreamsResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); + assertThat(getDataStreamsResponse.getDataStreams(), hasSize(4)); + getDataStreamsResponse.getDataStreams().sort(Comparator.comparing(DataStream::getName)); + assertThat(getDataStreamsResponse.getDataStreams().get(0).getName(), equalTo("logs-foobar")); + assertThat(getDataStreamsResponse.getDataStreams().get(1).getName(), equalTo("logs-foobaz")); + assertThat(getDataStreamsResponse.getDataStreams().get(2).getName(), equalTo("logs-foobaz2")); + assertThat(getDataStreamsResponse.getDataStreams().get(3).getName(), equalTo("logs-foobaz3")); + + GetIndexResponse getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices("logs-bar*")).actionGet(); + assertThat(getIndexResponse.getIndices(), arrayWithSize(4)); + assertThat(getIndexResponse.getIndices(), hasItemInArray("logs-barbaz")); + assertThat(getIndexResponse.getIndices(), hasItemInArray("logs-barfoo")); + assertThat(getIndexResponse.getIndices(), hasItemInArray("logs-barfoo2")); + assertThat(getIndexResponse.getIndices(), hasItemInArray("logs-barfoo3")); + + DeleteIndexTemplateV2Action.Request deleteTemplateRequest = new DeleteIndexTemplateV2Action.Request("*"); + client().execute(DeleteIndexTemplateV2Action.INSTANCE, deleteTemplateRequest).actionGet(); + } + + public void testAutoCreateV1TemplateNoDataStream() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build(); + + PutIndexTemplateRequest v1Request = new PutIndexTemplateRequest("logs-foo"); + v1Request.patterns(Collections.singletonList("logs-foo*")); + v1Request.settings(settings); + v1Request.order(Integer.MAX_VALUE); // in order to avoid number_of_replicas being overwritten by random_template + client().admin().indices().putTemplate(v1Request).actionGet(); + + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{}", XContentType.JSON)); + BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); + assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); + + GetDataStreamsAction.Request getDataStreamRequest = new GetDataStreamsAction.Request("*"); + GetDataStreamsAction.Response getDataStreamsResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); + assertThat(getDataStreamsResponse.getDataStreams(), hasSize(0)); + + GetIndexResponse getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices("logs-foobar")).actionGet(); + assertThat(getIndexResponse.getIndices(), arrayWithSize(1)); + assertThat(getIndexResponse.getIndices(), hasItemInArray("logs-foobar")); + assertThat(getIndexResponse.getSettings().get("logs-foobar").get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS), equalTo("0")); + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index c38876dae95..56c235656ae 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -21,19 +21,31 @@ package org.elasticsearch.action.admin.indices.create; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.action.support.ActiveShardsObserver; import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexTemplateV2; +import org.elasticsearch.cluster.metadata.IndexTemplateV2.DataStreamTemplate; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService; +import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Priority; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.concurrent.atomic.AtomicReference; /** * Api that auto creates an index that originate from requests that write into an index that doesn't yet exist. @@ -49,14 +61,19 @@ public final class AutoCreateAction extends ActionType { public static final class TransportAction extends TransportMasterNodeAction { + private final ActiveShardsObserver activeShardsObserver; private final MetadataCreateIndexService createIndexService; + private final MetadataCreateDataStreamService metadataCreateDataStreamService; @Inject public TransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - MetadataCreateIndexService createIndexService) { + MetadataCreateIndexService createIndexService, + MetadataCreateDataStreamService metadataCreateDataStreamService) { super(NAME, transportService, clusterService, threadPool, actionFilters, CreateIndexRequest::new, indexNameExpressionResolver); + this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool); this.createIndexService = createIndexService; + this.metadataCreateDataStreamService = metadataCreateDataStreamService; } @Override @@ -72,8 +89,55 @@ public final class AutoCreateAction extends ActionType { @Override protected void masterOperation(CreateIndexRequest request, ClusterState state, - ActionListener listener) throws Exception { - TransportCreateIndexAction.innerCreateIndex(request, listener, indexNameExpressionResolver, createIndexService); + ActionListener finalListener) { + AtomicReference indexNameRef = new AtomicReference<>(); + ActionListener listener = ActionListener.wrap( + response -> { + String indexName = indexNameRef.get(); + assert indexName != null; + if (response.isAcknowledged()) { + activeShardsObserver.waitForActiveShards( + new String[]{indexName}, + ActiveShardCount.DEFAULT, + request.timeout(), + shardsAcked -> { + finalListener.onResponse(new CreateIndexResponse(true, shardsAcked, indexName)); + }, + finalListener::onFailure + ); + } else { + finalListener.onResponse(new CreateIndexResponse(false, false, indexName)); + } + }, + finalListener::onFailure + ); + clusterService.submitStateUpdateTask("auto create [" + request.index() + "]", + new AckedClusterStateUpdateTask(Priority.URGENT, request, listener) { + + @Override + protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { + return new ClusterStateUpdateResponse(acknowledged); + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + DataStreamTemplate dataStreamTemplate = resolveAutoCreateDataStream(request, currentState.metadata()); + if (dataStreamTemplate != null) { + CreateDataStreamClusterStateUpdateRequest createRequest = new CreateDataStreamClusterStateUpdateRequest( + request.index(), dataStreamTemplate.getTimestampField(), request.masterNodeTimeout(), request.timeout()); + ClusterState clusterState = metadataCreateDataStreamService.createDataStream(createRequest, currentState); + indexNameRef.set(clusterState.metadata().dataStreams().get(request.index()).getIndices().get(0).getName()); + return clusterState; + } else { + String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index()); + indexNameRef.set(indexName); + CreateIndexClusterStateUpdateRequest updateRequest = + new CreateIndexClusterStateUpdateRequest(request.cause(), indexName, request.index()) + .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout()); + return createIndexService.applyCreateIndexRequest(currentState, updateRequest, false); + } + } + }); } @Override @@ -82,4 +146,16 @@ public final class AutoCreateAction extends ActionType { } } + static DataStreamTemplate resolveAutoCreateDataStream(CreateIndexRequest request, Metadata metadata) { + String v2Template = MetadataIndexTemplateService.findV2Template(metadata, request.index(), false); + if (v2Template != null) { + IndexTemplateV2 indexTemplateV2 = metadata.templatesV2().get(v2Template); + if (indexTemplateV2.getDataStreamTemplate() != null) { + return indexTemplateV2.getDataStreamTemplate(); + } + } + + return null; + } + } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java index 34867ee037e..b6f0aa55582 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java @@ -70,20 +70,14 @@ public class TransportCreateIndexAction extends TransportMasterNodeAction listener) { - if (request.cause().length() == 0) { - request.cause("api"); + String cause = request.cause(); + if (cause.length() == 0) { + cause = "api"; } - innerCreateIndex(request, listener, indexNameExpressionResolver, createIndexService); - } - - static void innerCreateIndex(CreateIndexRequest request, - ActionListener listener, - IndexNameExpressionResolver indexNameExpressionResolver, - MetadataCreateIndexService createIndexService) { final String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index()); final CreateIndexClusterStateUpdateRequest updateRequest = - new CreateIndexClusterStateUpdateRequest(request.cause(), indexName, request.index()) + new CreateIndexClusterStateUpdateRequest(cause, indexName, request.index()) .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout()) .settings(request.settings()).mappings(request.mappings()) .aliases(request.aliases()) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java index 6fed62f0ca5..58ce10219db 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java @@ -18,46 +18,33 @@ */ package org.elasticsearch.action.admin.indices.datastream; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.ValidateActions; -import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; +import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService; +import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.List; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.io.IOException; -import java.util.Locale; import java.util.Objects; public class CreateDataStreamAction extends ActionType { - private static final Logger logger = LogManager.getLogger(CreateDataStreamAction.class); - public static final CreateDataStreamAction INSTANCE = new CreateDataStreamAction(); public static final String NAME = "indices:admin/data_stream/create"; @@ -65,7 +52,7 @@ public class CreateDataStreamAction extends ActionType { super(NAME, AcknowledgedResponse::new); } - public static class Request extends MasterNodeRequest { + public static class Request extends AcknowledgedRequest { private final String name; private String timestampFieldName; @@ -120,14 +107,14 @@ public class CreateDataStreamAction extends ActionType { public static class TransportAction extends TransportMasterNodeAction { - private final MetadataCreateIndexService metadataCreateIndexService; + private final MetadataCreateDataStreamService metadataCreateDataStreamService; @Inject public TransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - MetadataCreateIndexService metadataCreateIndexService) { + MetadataCreateDataStreamService metadataCreateDataStreamService) { super(NAME, transportService, clusterService, threadPool, actionFilters, Request::new, indexNameExpressionResolver); - this.metadataCreateIndexService = metadataCreateIndexService; + this.metadataCreateDataStreamService = metadataCreateDataStreamService; } @Override @@ -143,60 +130,13 @@ public class CreateDataStreamAction extends ActionType { @Override protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { - clusterService.submitStateUpdateTask("create-data-stream [" + request.name + "]", - new ClusterStateUpdateTask(Priority.HIGH) { - - @Override - public TimeValue timeout() { - return request.masterNodeTimeout(); - } - - @Override - public void onFailure(String source, Exception e) { - listener.onFailure(e); - } - - @Override - public ClusterState execute(ClusterState currentState) throws Exception { - return createDataStream(metadataCreateIndexService, currentState, request); - } - - @Override - public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - listener.onResponse(new AcknowledgedResponse(true)); - } - }); - } - - static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIndexService, - ClusterState currentState, - Request request) throws Exception { - if (currentState.metadata().dataStreams().containsKey(request.name)) { - throw new IllegalArgumentException("data_stream [" + request.name + "] already exists"); - } - - MetadataCreateIndexService.validateIndexOrAliasName(request.name, - (s1, s2) -> new IllegalArgumentException("data_stream [" + s1 + "] " + s2)); - - if (request.name.toLowerCase(Locale.ROOT).equals(request.name) == false) { - throw new IllegalArgumentException("data_stream [" + request.name + "] must be lowercase"); - } - if (request.name.startsWith(".")) { - throw new IllegalArgumentException("data_stream [" + request.name + "] must not start with '.'"); - } - - String firstBackingIndexName = DataStream.getBackingIndexName(request.name, 1); - CreateIndexClusterStateUpdateRequest createIndexRequest = - new CreateIndexClusterStateUpdateRequest("initialize_data_stream", firstBackingIndexName, firstBackingIndexName) - .settings(Settings.builder().put("index.hidden", true).build()); - currentState = metadataCreateIndexService.applyCreateIndexRequest(currentState, createIndexRequest, false); - IndexMetadata firstBackingIndex = currentState.metadata().index(firstBackingIndexName); - assert firstBackingIndex != null; - - Metadata.Builder builder = Metadata.builder(currentState.metadata()).put( - new DataStream(request.name, request.timestampFieldName, List.of(firstBackingIndex.getIndex()))); - logger.info("adding data stream [{}]", request.name); - return ClusterState.builder(currentState).metadata(builder).build(); + CreateDataStreamClusterStateUpdateRequest updateRequest = new CreateDataStreamClusterStateUpdateRequest( + request.name, + request.timestampFieldName, + request.masterNodeTimeout(), + request.timeout() + ); + metadataCreateDataStreamService.createDataStream(updateRequest, listener); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index 98b7510f149..1be68517c0a 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -69,7 +69,7 @@ public class TransportFieldCapabilitiesAction extends HandledTransportAction listener) { final ClusterState clusterState = clusterService.state(); final Map remoteClusterIndices = remoteClusterService.groupIndices(request.indicesOptions(), - request.indices(), idx -> indexNameExpressionResolver.hasIndexOrAlias(idx, clusterState)); + request.indices(), idx -> indexNameExpressionResolver.hasIndexAbstraction(idx, clusterState)); final OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); final String[] concreteIndices; if (localIndices == null) { diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 295eff34638..2f45e92c228 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -207,7 +207,7 @@ public class TransportSearchAction extends HandledTransportAction remoteClusterIndices = remoteClusterService.groupIndices(searchRequest.indicesOptions(), - searchRequest.indices(), idx -> indexNameExpressionResolver.hasIndexOrAlias(idx, clusterState)); + searchRequest.indices(), idx -> indexNameExpressionResolver.hasIndexAbstraction(idx, clusterState)); OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); if (remoteClusterIndices.isEmpty()) { executeLocalSearch(task, timeProvider, searchRequest, localIndices, clusterState, listener); diff --git a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java index f530a81e51d..2483d9fa529 100644 --- a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java +++ b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java @@ -67,7 +67,7 @@ public final class AutoCreateIndex { * @throws IndexNotFoundException if the index doesn't exist and shouldn't be auto created */ public boolean shouldAutoCreate(String index, ClusterState state) { - if (resolver.hasIndexOrAlias(index, state)) { + if (resolver.hasIndexAbstraction(index, state)) { return false; } // One volatile read, so that all checks are done against the same instance: diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index cc5669cb394..0a4ebb6f1e2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -364,11 +364,12 @@ public class IndexNameExpressionResolver { } /** - * @return whether the specified alias or index exists. If the alias or index contains datemath then that is resolved too. + * @return whether the specified index, data stream or alias exists. + * If the data stream, index or alias contains date math then that is resolved too. */ - public boolean hasIndexOrAlias(String aliasOrIndex, ClusterState state) { - Context context = new Context(state, IndicesOptions.lenientExpandOpen()); - String resolvedAliasOrIndex = dateMathExpressionResolver.resolveExpression(aliasOrIndex, context); + public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state) { + Context context = new Context(state, IndicesOptions.lenientExpandOpen(), false, false, true); + String resolvedAliasOrIndex = dateMathExpressionResolver.resolveExpression(indexAbstraction, context); return state.metadata().getIndicesLookup().containsKey(resolvedAliasOrIndex); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java index bf4279d5ab7..66ecdf2b2a1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.Nullable; @@ -26,6 +27,7 @@ import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -49,6 +51,7 @@ public class IndexTemplateV2 extends AbstractDiffable implement private static final ParseField COMPOSED_OF = new ParseField("composed_of"); private static final ParseField VERSION = new ParseField("version"); private static final ParseField METADATA = new ParseField("_meta"); + private static final ParseField DATA_STREAM = new ParseField("data_stream"); @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("index_template", false, @@ -57,7 +60,8 @@ public class IndexTemplateV2 extends AbstractDiffable implement (List) a[2], (Long) a[3], (Long) a[4], - (Map) a[5])); + (Map) a[5], + (DataStreamTemplate) a[6])); static { PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INDEX_PATTERNS); @@ -66,6 +70,7 @@ public class IndexTemplateV2 extends AbstractDiffable implement PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), PRIORITY); PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VERSION); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), DataStreamTemplate.PARSER, DATA_STREAM); } private final List indexPatterns; @@ -79,6 +84,8 @@ public class IndexTemplateV2 extends AbstractDiffable implement private final Long version; @Nullable private final Map metadata; + @Nullable + private final DataStreamTemplate dataStreamTemplate; static Diff readITV2DiffFrom(StreamInput in) throws IOException { return AbstractDiffable.readDiffFrom(IndexTemplateV2::new, in); @@ -90,12 +97,19 @@ public class IndexTemplateV2 extends AbstractDiffable implement public IndexTemplateV2(List indexPatterns, @Nullable Template template, @Nullable List componentTemplates, @Nullable Long priority, @Nullable Long version, @Nullable Map metadata) { + this(indexPatterns, template, componentTemplates, priority, version, metadata, null); + } + + public IndexTemplateV2(List indexPatterns, @Nullable Template template, @Nullable List componentTemplates, + @Nullable Long priority, @Nullable Long version, @Nullable Map metadata, + @Nullable DataStreamTemplate dataStreamTemplate) { this.indexPatterns = indexPatterns; this.template = template; this.componentTemplates = componentTemplates; this.priority = priority; this.version = version; this.metadata = metadata; + this.dataStreamTemplate = dataStreamTemplate; } public IndexTemplateV2(StreamInput in) throws IOException { @@ -109,6 +123,11 @@ public class IndexTemplateV2 extends AbstractDiffable implement this.priority = in.readOptionalVLong(); this.version = in.readOptionalVLong(); this.metadata = in.readMap(); + if (in.getVersion().onOrAfter(Version.V_7_9_0)) { + this.dataStreamTemplate = in.readOptionalWriteable(DataStreamTemplate::new); + } else { + this.dataStreamTemplate = null; + } } public List indexPatterns() { @@ -146,6 +165,10 @@ public class IndexTemplateV2 extends AbstractDiffable implement return metadata; } + public DataStreamTemplate getDataStreamTemplate() { + return dataStreamTemplate; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeStringCollection(this.indexPatterns); @@ -159,6 +182,9 @@ public class IndexTemplateV2 extends AbstractDiffable implement out.writeOptionalVLong(this.priority); out.writeOptionalVLong(this.version); out.writeMap(this.metadata); + if (out.getVersion().onOrAfter(Version.V_7_9_0)) { + out.writeOptionalWriteable(dataStreamTemplate); + } } @Override @@ -180,13 +206,17 @@ public class IndexTemplateV2 extends AbstractDiffable implement if (this.metadata != null) { builder.field(METADATA.getPreferredName(), metadata); } + if (this.dataStreamTemplate != null) { + builder.field(DATA_STREAM.getPreferredName(), dataStreamTemplate); + } builder.endObject(); return builder; } @Override public int hashCode() { - return Objects.hash(this.indexPatterns, this.template, this.componentTemplates, this.priority, this.version, this.metadata); + return Objects.hash(this.indexPatterns, this.template, this.componentTemplates, this.priority, this.version, + this.metadata, this.dataStreamTemplate); } @Override @@ -203,11 +233,64 @@ public class IndexTemplateV2 extends AbstractDiffable implement Objects.equals(this.componentTemplates, other.componentTemplates) && Objects.equals(this.priority, other.priority) && Objects.equals(this.version, other.version) && - Objects.equals(this.metadata, other.metadata); + Objects.equals(this.metadata, other.metadata) && + Objects.equals(this.dataStreamTemplate, other.dataStreamTemplate); } @Override public String toString() { return Strings.toString(this); } + + public static class DataStreamTemplate implements Writeable, ToXContentObject { + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "data_stream_template", + args -> new DataStreamTemplate((String) args[0]) + ); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), DataStream.TIMESTAMP_FIELD_FIELD); + } + + private final String timestampField; + + public DataStreamTemplate(String timestampField) { + this.timestampField = timestampField; + } + + public String getTimestampField() { + return timestampField; + } + + DataStreamTemplate(StreamInput in) throws IOException { + this(in.readString()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(timestampField); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(DataStream.TIMESTAMP_FIELD_FIELD.getPreferredName(), timestampField); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DataStreamTemplate that = (DataStreamTemplate) o; + return timestampField.equals(that.timestampField); + } + + @Override + public int hashCode() { + return Objects.hash(timestampField); + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java new file mode 100644 index 00000000000..f054730e033 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -0,0 +1,148 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.metadata; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; +import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.action.support.ActiveShardsObserver; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.AckedClusterStateUpdateTask; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; +import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.Collections; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicReference; + +public class MetadataCreateDataStreamService { + + private static final Logger logger = LogManager.getLogger(MetadataCreateDataStreamService.class); + + private final ClusterService clusterService; + private final ActiveShardsObserver activeShardsObserver; + private final MetadataCreateIndexService metadataCreateIndexService; + + public MetadataCreateDataStreamService(ThreadPool threadPool, + ClusterService clusterService, + MetadataCreateIndexService metadataCreateIndexService) { + this.clusterService = clusterService; + this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool); + this.metadataCreateIndexService = metadataCreateIndexService; + } + + public void createDataStream(CreateDataStreamClusterStateUpdateRequest request, + ActionListener finalListener) { + AtomicReference firstBackingIndexRef = new AtomicReference<>(); + ActionListener listener = ActionListener.wrap( + response -> { + if (response.isAcknowledged()) { + String firstBackingIndexName = firstBackingIndexRef.get(); + assert firstBackingIndexName != null; + activeShardsObserver.waitForActiveShards( + new String[]{firstBackingIndexName}, + ActiveShardCount.DEFAULT, + request.masterNodeTimeout(), + shardsAcked -> { + finalListener.onResponse(new AcknowledgedResponse(true)); + }, + finalListener::onFailure); + } else { + finalListener.onResponse(new AcknowledgedResponse(false)); + } + }, + finalListener::onFailure + ); + clusterService.submitStateUpdateTask("create-data-stream [" + request.name + "]", + new AckedClusterStateUpdateTask(Priority.HIGH, request, listener) { + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + ClusterState clusterState = createDataStream(metadataCreateIndexService, currentState, request); + firstBackingIndexRef.set(clusterState.metadata().dataStreams().get(request.name).getIndices().get(0).getName()); + return clusterState; + } + + @Override + protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { + return new ClusterStateUpdateResponse(acknowledged); + } + }); + } + + public ClusterState createDataStream(CreateDataStreamClusterStateUpdateRequest request, ClusterState current) throws Exception { + return createDataStream(metadataCreateIndexService, current, request); + } + + public static final class CreateDataStreamClusterStateUpdateRequest extends ClusterStateUpdateRequest { + + private final String name; + private final String timestampFieldName; + + public CreateDataStreamClusterStateUpdateRequest(String name, + String timestampFieldName, + TimeValue masterNodeTimeout, + TimeValue timeout) { + this.name = name; + this.timestampFieldName = timestampFieldName; + masterNodeTimeout(masterNodeTimeout); + ackTimeout(timeout); + } + } + + static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIndexService, + ClusterState currentState, + CreateDataStreamClusterStateUpdateRequest request) throws Exception { + if (currentState.metadata().dataStreams().containsKey(request.name)) { + throw new IllegalArgumentException("data_stream [" + request.name + "] already exists"); + } + + MetadataCreateIndexService.validateIndexOrAliasName(request.name, + (s1, s2) -> new IllegalArgumentException("data_stream [" + s1 + "] " + s2)); + + if (request.name.toLowerCase(Locale.ROOT).equals(request.name) == false) { + throw new IllegalArgumentException("data_stream [" + request.name + "] must be lowercase"); + } + if (request.name.startsWith(".")) { + throw new IllegalArgumentException("data_stream [" + request.name + "] must not start with '.'"); + } + + String firstBackingIndexName = DataStream.getBackingIndexName(request.name, 1); + CreateIndexClusterStateUpdateRequest createIndexRequest = + new CreateIndexClusterStateUpdateRequest("initialize_data_stream", firstBackingIndexName, firstBackingIndexName) + .settings(Settings.builder().put("index.hidden", true).build()); + currentState = metadataCreateIndexService.applyCreateIndexRequest(currentState, createIndexRequest, false); + IndexMetadata firstBackingIndex = currentState.metadata().index(firstBackingIndexName); + assert firstBackingIndex != null; + + Metadata.Builder builder = Metadata.builder(currentState.metadata()).put( + new DataStream(request.name, request.timestampFieldName, Collections.singletonList(firstBackingIndex.getIndex()))); + logger.info("adding data stream [{}]", request.name); + return ClusterState.builder(currentState).metadata(builder).build(); + } + +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 21a070e44d5..811177e63f1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -400,7 +400,7 @@ public class MetadataIndexTemplateService { final Template finalTemplate = new Template(finalSettings, stringMappings == null ? null : new CompressedXContent(stringMappings), innerTemplate.aliases()); finalIndexTemplate = new IndexTemplateV2(template.indexPatterns(), finalTemplate, template.composedOf(), - template.priority(), template.version(), template.metadata()); + template.priority(), template.version(), template.metadata(), template.getDataStreamTemplate()); } validate(name, finalIndexTemplate); diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index c3b69015eae..60df04a4eaf 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -51,6 +51,7 @@ import org.elasticsearch.cluster.action.index.MappingUpdatedAction; import org.elasticsearch.cluster.metadata.AliasValidator; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataIndexUpgradeService; import org.elasticsearch.cluster.metadata.TemplateUpgradeService; @@ -479,6 +480,9 @@ public class Node implements Closeable { systemIndexDescriptors, forbidPrivateIndexSettings); + final MetadataCreateDataStreamService metadataCreateDataStreamService = + new MetadataCreateDataStreamService(threadPool, clusterService, metadataCreateIndexService); + final SetOnce repositoriesServiceReference = new SetOnce<>(); Collection pluginComponents = pluginsService.filterPlugins(Plugin.class).stream() .flatMap(p -> p.createComponents(client, clusterService, threadPool, resourceWatcherService, @@ -590,6 +594,7 @@ public class Node implements Closeable { b.bind(IndicesService.class).toInstance(indicesService); b.bind(AliasValidator.class).toInstance(aliasValidator); b.bind(MetadataCreateIndexService.class).toInstance(metadataCreateIndexService); + b.bind(MetadataCreateDataStreamService.class).toInstance(metadataCreateDataStreamService); b.bind(SearchService.class).toInstance(searchService); b.bind(SearchTransportService.class).toInstance(searchTransportService); b.bind(SearchPhaseController.class).toInstance(new SearchPhaseController( diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java new file mode 100644 index 00000000000..6ebe7d3fb78 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.create; + +import org.elasticsearch.cluster.metadata.IndexTemplateV2; +import org.elasticsearch.cluster.metadata.IndexTemplateV2.DataStreamTemplate; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.test.ESTestCase; + +import java.util.Collections; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class AutoCreateActionTests extends ESTestCase { + + public void testResolveAutoCreateDataStreams() { + Metadata metadata; + { + Metadata.Builder mdBuilder = new Metadata.Builder(); + DataStreamTemplate dataStreamTemplate = new DataStreamTemplate("@timestamp"); + mdBuilder.put("1", new IndexTemplateV2(Collections.singletonList("legacy-logs-*"), null, null, 10L, null, null, null)); + mdBuilder.put("2", new IndexTemplateV2(Collections.singletonList("logs-*"), null, null, 20L, null, null, dataStreamTemplate)); + mdBuilder.put("3", + new IndexTemplateV2(Collections.singletonList("logs-foobar"), null, null, 30L, null, null, dataStreamTemplate)); + metadata = mdBuilder.build(); + } + + CreateIndexRequest request = new CreateIndexRequest("logs-foobar"); + DataStreamTemplate result = AutoCreateAction.resolveAutoCreateDataStream(request, metadata); + assertThat(result, notNullValue()); + assertThat(result.getTimestampField(), equalTo("@timestamp")); + + request = new CreateIndexRequest("logs-barbaz"); + result = AutoCreateAction.resolveAutoCreateDataStream(request, metadata); + assertThat(result, notNullValue()); + assertThat(result.getTimestampField(), equalTo("@timestamp")); + + // An index that matches with a template without a data steam definition + request = new CreateIndexRequest("legacy-logs-foobaz"); + result = AutoCreateAction.resolveAutoCreateDataStream(request, metadata); + assertThat(result, nullValue()); + + // An index that doesn't match with an index template + request = new CreateIndexRequest("my-index"); + result = AutoCreateAction.resolveAutoCreateDataStream(request, metadata); + assertThat(result, nullValue()); + } + +} diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java index ade22188832..b4b7326a6b5 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java @@ -18,30 +18,13 @@ */ package org.elasticsearch.action.admin.indices.datastream; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction.Request; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; -import org.elasticsearch.common.collect.List; -import org.elasticsearch.common.collect.Map; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import static org.elasticsearch.cluster.DataStreamTestHelper.createFirstBackingIndex; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.notNullValue; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyBoolean; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class CreateDataStreamRequestTests extends AbstractWireSerializingTestCase { @@ -72,82 +55,4 @@ public class CreateDataStreamRequestTests extends AbstractWireSerializingTestCas assertThat(e.validationErrors().get(0), containsString("timestamp field name is missing")); } - public void testCreateDataStream() throws Exception { - final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); - final String dataStreamName = "my-data-stream"; - ClusterState cs = ClusterState.builder(new ClusterName("_name")).build(); - CreateDataStreamAction.Request req = new CreateDataStreamAction.Request(dataStreamName); - ClusterState newState = CreateDataStreamAction.TransportAction.createDataStream(metadataCreateIndexService, cs, req); - assertThat(newState.metadata().dataStreams().size(), equalTo(1)); - assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); - assertThat(newState.metadata().index(DataStream.getBackingIndexName(dataStreamName, 1)), notNullValue()); - assertThat(newState.metadata().index(DataStream.getBackingIndexName(dataStreamName, 1)).getSettings().get("index.hidden"), - equalTo("true")); - } - - public void testCreateDuplicateDataStream() throws Exception { - final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); - final String dataStreamName = "my-data-stream"; - IndexMetadata idx = createFirstBackingIndex(dataStreamName).build(); - DataStream existingDataStream = new DataStream(dataStreamName, "timestamp", List.of(idx.getIndex())); - ClusterState cs = ClusterState.builder(new ClusterName("_name")) - .metadata(Metadata.builder().dataStreams(Map.of(dataStreamName, existingDataStream)).build()).build(); - CreateDataStreamAction.Request req = new CreateDataStreamAction.Request(dataStreamName); - - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> CreateDataStreamAction.TransportAction.createDataStream(metadataCreateIndexService, cs, req)); - assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] already exists")); - } - - public void testCreateDataStreamWithInvalidName() throws Exception { - final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); - final String dataStreamName = "_My-da#ta- ,stream-"; - ClusterState cs = ClusterState.builder(new ClusterName("_name")).build(); - CreateDataStreamAction.Request req = new CreateDataStreamAction.Request(dataStreamName); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> CreateDataStreamAction.TransportAction.createDataStream(metadataCreateIndexService, cs, req)); - assertThat(e.getMessage(), containsString("must not contain the following characters")); - } - - public void testCreateDataStreamWithUppercaseCharacters() throws Exception { - final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); - final String dataStreamName = "MAY_NOT_USE_UPPERCASE"; - ClusterState cs = ClusterState.builder(new ClusterName("_name")).build(); - CreateDataStreamAction.Request req = new CreateDataStreamAction.Request(dataStreamName); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> CreateDataStreamAction.TransportAction.createDataStream(metadataCreateIndexService, cs, req)); - assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] must be lowercase")); - } - - public void testCreateDataStreamStartingWithPeriod() throws Exception { - final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); - final String dataStreamName = ".may_not_start_with_period"; - ClusterState cs = ClusterState.builder(new ClusterName("_name")).build(); - CreateDataStreamAction.Request req = new CreateDataStreamAction.Request(dataStreamName); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> CreateDataStreamAction.TransportAction.createDataStream(metadataCreateIndexService, cs, req)); - assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] must not start with '.'")); - } - - private static MetadataCreateIndexService getMetadataCreateIndexService() throws Exception { - MetadataCreateIndexService s = mock(MetadataCreateIndexService.class); - when(s.applyCreateIndexRequest(any(ClusterState.class), any(CreateIndexClusterStateUpdateRequest.class), anyBoolean())) - .thenAnswer(mockInvocation -> { - ClusterState currentState = (ClusterState) mockInvocation.getArguments()[0]; - CreateIndexClusterStateUpdateRequest request = (CreateIndexClusterStateUpdateRequest) mockInvocation.getArguments()[1]; - - Metadata.Builder b = Metadata.builder(currentState.metadata()) - .put(IndexMetadata.builder(request.index()) - .settings(Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(request.settings()) - .build()) - .numberOfShards(1) - .numberOfReplicas(1) - .build(), false); - return ClusterState.builder(currentState).metadata(b.build()).build(); - }); - - return s; - } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 6e950822462..fc896e95f52 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -353,7 +353,7 @@ public class MetadataRolloverServiceTests extends ESTestCase { aliases.put("foo-write", AliasMetadata.builder("foo-write").build()); aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build()); final IndexTemplateV2 template = new IndexTemplateV2(Arrays.asList("foo-*", "bar-*"), new Template(null, null, aliases), - null, null, null, null); + null, null, null, null, null); final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false) .put("test-template", template).build(); @@ -370,7 +370,7 @@ public class MetadataRolloverServiceTests extends ESTestCase { aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build()); final ComponentTemplate ct = new ComponentTemplate(new Template(null, null, aliases), null, null); final IndexTemplateV2 template = new IndexTemplateV2(Arrays.asList("foo-*", "bar-*"), null, - Collections.singletonList("ct"), null, null, null); + Collections.singletonList("ct"), null, null, null, null); final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false) .put("ct", ct) @@ -405,7 +405,7 @@ public class MetadataRolloverServiceTests extends ESTestCase { aliases.put("foo-write", AliasMetadata.builder("foo-write").build()); aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build()); final IndexTemplateV2 template = new IndexTemplateV2(Collections.singletonList("*"), new Template(null, null, aliases), - null, null, null, null); + null, null, null, null, null); final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false) .put("test-template", template).build(); @@ -426,7 +426,7 @@ public class MetadataRolloverServiceTests extends ESTestCase { aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build()); final ComponentTemplate ct = new ComponentTemplate(new Template(null, null, aliases), null, null); final IndexTemplateV2 template = new IndexTemplateV2(Collections.singletonList("*"), null, - Collections.singletonList("ct"), null, null, null); + Collections.singletonList("ct"), null, null, null, null); final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false) .put("ct", ct) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java index 6917b95d81a..f48cf706d4d 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java @@ -64,7 +64,7 @@ public class SimulateIndexTemplateRequestTests extends AbstractWireSerializingTe public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null); IndexTemplateV2 globalTemplate = new IndexTemplateV2(org.elasticsearch.common.collect.List.of("*"), template, null, null, null, - null); + null, null); PutIndexTemplateV2Action.Request request = new PutIndexTemplateV2Action.Request("test"); request.indexTemplate(globalTemplate); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateV2RequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateV2RequestTests.java index e50f62d18bc..b29158d12e0 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateV2RequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateV2RequestTests.java @@ -58,7 +58,7 @@ public class PutIndexTemplateV2RequestTests extends AbstractWireSerializingTestC public void testPutGlobalTemplatesCannotHaveHiddenIndexSetting() { Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null); IndexTemplateV2 globalTemplate = new IndexTemplateV2(org.elasticsearch.common.collect.List.of("*"), template, null, null, null, - null); + null, null); PutIndexTemplateV2Action.Request request = new PutIndexTemplateV2Action.Request("test"); request.indexTemplate(globalTemplate); @@ -84,7 +84,7 @@ public class PutIndexTemplateV2RequestTests extends AbstractWireSerializingTestC public void testValidationOfPriority() { PutIndexTemplateV2Action.Request req = new PutIndexTemplateV2Action.Request("test"); - req.indexTemplate(new IndexTemplateV2(Arrays.asList("foo", "bar"), null, null, -5L, null, null)); + req.indexTemplate(new IndexTemplateV2(Arrays.asList("foo", "bar"), null, null, -5L, null, null, null)); ActionRequestValidationException validationException = req.validate(); assertThat(validationException, is(notNullValue())); List validationErrors = validationException.validationErrors(); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index 9cf1f314233..d531d4ef1f2 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -580,7 +580,7 @@ public class TransportBulkActionIngestTests extends ESTestCase { IndexTemplateV2 t1 = new IndexTemplateV2(Collections.singletonList("missing_*"), new Template(Settings.builder().put(IndexSettings.DEFAULT_PIPELINE.getKey(), "pipeline2").build(), null, null), - null, null, null, null); + null, null, null, null, null); ClusterState state = clusterService.state(); Metadata metadata = Metadata.builder() diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index 301d0c43776..0e147eb122b 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -289,4 +289,5 @@ public class TransportBulkActionTests extends ESTestCase { assertThat(indexRequest.getFinalPipeline(), equalTo("final-pipeline")); } } + } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Tests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Tests.java index 121c10a5338..e6b027798f1 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Tests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Tests.java @@ -87,6 +87,8 @@ public class IndexTemplateV2Tests extends AbstractDiffableSerializationTestCase< meta = randomMeta(); } + IndexTemplateV2.DataStreamTemplate dataStreamTemplate = randomDataStreamTemplate(); + List indexPatterns = randomList(1, 4, () -> randomAlphaOfLength(4)); List componentTemplates = randomList(0, 10, () -> randomAlphaOfLength(5)); return new IndexTemplateV2(indexPatterns, @@ -94,7 +96,8 @@ public class IndexTemplateV2Tests extends AbstractDiffableSerializationTestCase< componentTemplates, randomBoolean() ? null : randomNonNegativeLong(), randomBoolean() ? null : randomNonNegativeLong(), - meta); + meta, + dataStreamTemplate); } private static Map randomAliases() { @@ -137,25 +140,34 @@ public class IndexTemplateV2Tests extends AbstractDiffableSerializationTestCase< } } + private static IndexTemplateV2.DataStreamTemplate randomDataStreamTemplate() { + if (randomBoolean()) { + return null; + } else { + return new IndexTemplateV2.DataStreamTemplate(randomAlphaOfLength(8)); + } + } + @Override protected IndexTemplateV2 mutateInstance(IndexTemplateV2 orig) throws IOException { return mutateTemplate(orig); } public static IndexTemplateV2 mutateTemplate(IndexTemplateV2 orig) { - switch (randomIntBetween(0, 5)) { + switch (randomIntBetween(0, 6)) { case 0: List newIndexPatterns = randomValueOtherThan(orig.indexPatterns(), () -> randomList(1, 4, () -> randomAlphaOfLength(4))); return new IndexTemplateV2(newIndexPatterns, orig.template(), orig.composedOf(), - orig.priority(), orig.version(), orig.metadata()); + orig.priority(), orig.version(), orig.metadata(), orig.getDataStreamTemplate()); case 1: return new IndexTemplateV2(orig.indexPatterns(), randomValueOtherThan(orig.template(), () -> new Template(randomSettings(), randomMappings(), randomAliases())), orig.composedOf(), orig.priority(), orig.version(), - orig.metadata()); + orig.metadata(), + orig.getDataStreamTemplate()); case 2: List newComposedOf = randomValueOtherThan(orig.composedOf(), () -> randomList(0, 10, () -> randomAlphaOfLength(5))); @@ -164,28 +176,40 @@ public class IndexTemplateV2Tests extends AbstractDiffableSerializationTestCase< newComposedOf, orig.priority(), orig.version(), - orig.metadata()); + orig.metadata(), + orig.getDataStreamTemplate()); case 3: return new IndexTemplateV2(orig.indexPatterns(), orig.template(), orig.composedOf(), randomValueOtherThan(orig.priority(), ESTestCase::randomNonNegativeLong), orig.version(), - orig.metadata()); + orig.metadata(), + orig.getDataStreamTemplate()); case 4: return new IndexTemplateV2(orig.indexPatterns(), orig.template(), orig.composedOf(), orig.priority(), randomValueOtherThan(orig.version(), ESTestCase::randomNonNegativeLong), - orig.metadata()); + orig.metadata(), + orig.getDataStreamTemplate()); case 5: return new IndexTemplateV2(orig.indexPatterns(), orig.template(), orig.composedOf(), orig.priority(), orig.version(), - randomValueOtherThan(orig.metadata(), IndexTemplateV2Tests::randomMeta)); + randomValueOtherThan(orig.metadata(), IndexTemplateV2Tests::randomMeta), + orig.getDataStreamTemplate()); + case 6: + return new IndexTemplateV2(orig.indexPatterns(), + orig.template(), + orig.composedOf(), + orig.priority(), + orig.version(), + orig.metadata(), + randomValueOtherThan(orig.getDataStreamTemplate(), IndexTemplateV2Tests::randomDataStreamTemplate)); default: throw new IllegalStateException("illegal randomization branch"); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java new file mode 100644 index 00000000000..7ca14e34b23 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -0,0 +1,127 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.test.ESTestCase; + +import java.util.Collections; + +import static org.elasticsearch.cluster.DataStreamTestHelper.createFirstBackingIndex; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class MetadataCreateDataStreamServiceTests extends ESTestCase { + + public void testCreateDataStream() throws Exception { + final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); + final String dataStreamName = "my-data-stream"; + ClusterState cs = ClusterState.builder(new ClusterName("_name")).build(); + CreateDataStreamClusterStateUpdateRequest req = + new CreateDataStreamClusterStateUpdateRequest(dataStreamName, "@timestamp", TimeValue.ZERO, TimeValue.ZERO); + ClusterState newState = MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req); + assertThat(newState.metadata().dataStreams().size(), equalTo(1)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); + assertThat(newState.metadata().index(DataStream.getBackingIndexName(dataStreamName, 1)), notNullValue()); + assertThat(newState.metadata().index(DataStream.getBackingIndexName(dataStreamName, 1)).getSettings().get("index.hidden"), + equalTo("true")); + } + + public void testCreateDuplicateDataStream() throws Exception { + final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); + final String dataStreamName = "my-data-stream"; + IndexMetadata idx = createFirstBackingIndex(dataStreamName).build(); + DataStream existingDataStream = new DataStream(dataStreamName, "timestamp", Collections.singletonList(idx.getIndex())); + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .metadata(Metadata.builder().dataStreams(Collections.singletonMap(dataStreamName, existingDataStream)).build()).build(); + CreateDataStreamClusterStateUpdateRequest req = + new CreateDataStreamClusterStateUpdateRequest(dataStreamName, "@timestamp", TimeValue.ZERO, TimeValue.ZERO); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); + assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] already exists")); + } + + public void testCreateDataStreamWithInvalidName() throws Exception { + final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); + final String dataStreamName = "_My-da#ta- ,stream-"; + ClusterState cs = ClusterState.builder(new ClusterName("_name")).build(); + CreateDataStreamClusterStateUpdateRequest req = + new CreateDataStreamClusterStateUpdateRequest(dataStreamName, "@timestamp", TimeValue.ZERO, TimeValue.ZERO); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); + assertThat(e.getMessage(), containsString("must not contain the following characters")); + } + + public void testCreateDataStreamWithUppercaseCharacters() throws Exception { + final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); + final String dataStreamName = "MAY_NOT_USE_UPPERCASE"; + ClusterState cs = ClusterState.builder(new ClusterName("_name")).build(); + CreateDataStreamClusterStateUpdateRequest req = + new CreateDataStreamClusterStateUpdateRequest(dataStreamName, "@timestamp", TimeValue.ZERO, TimeValue.ZERO); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); + assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] must be lowercase")); + } + + public void testCreateDataStreamStartingWithPeriod() throws Exception { + final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); + final String dataStreamName = ".may_not_start_with_period"; + ClusterState cs = ClusterState.builder(new ClusterName("_name")).build(); + CreateDataStreamClusterStateUpdateRequest req = + new CreateDataStreamClusterStateUpdateRequest(dataStreamName, "@timestamp", TimeValue.ZERO, TimeValue.ZERO); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); + assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] must not start with '.'")); + } + + private static MetadataCreateIndexService getMetadataCreateIndexService() throws Exception { + MetadataCreateIndexService s = mock(MetadataCreateIndexService.class); + when(s.applyCreateIndexRequest(any(ClusterState.class), any(CreateIndexClusterStateUpdateRequest.class), anyBoolean())) + .thenAnswer(mockInvocation -> { + ClusterState currentState = (ClusterState) mockInvocation.getArguments()[0]; + CreateIndexClusterStateUpdateRequest request = (CreateIndexClusterStateUpdateRequest) mockInvocation.getArguments()[1]; + + Metadata.Builder b = Metadata.builder(currentState.metadata()) + .put(IndexMetadata.builder(request.index()) + .settings(Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(request.settings()) + .build()) + .numberOfShards(1) + .numberOfReplicas(1) + .build(), false); + return ClusterState.builder(currentState).metadata(b.build()).build(); + }); + + return s; + } + +} diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index d593fae231c..805eb75e15a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -1016,7 +1016,7 @@ public class MetadataCreateIndexServiceTests extends ESTestCase { boolean shouldBeText = randomBoolean(); List composedOf = shouldBeText ? Arrays.asList("ct2", "ct1") : Arrays.asList("ct1", "ct2"); logger.info("--> the {} analyzer should win ({})", shouldBeText ? "text" : "keyword", composedOf); - IndexTemplateV2 template = new IndexTemplateV2(Collections.singletonList("index"), null, composedOf, null, null, null); + IndexTemplateV2 template = new IndexTemplateV2(Collections.singletonList("index"), null, composedOf, null, null, null, null); ClusterState state = ClusterState.builder(ClusterState.EMPTY_STATE) .metadata(Metadata.builder(Metadata.EMPTY_METADATA) @@ -1075,7 +1075,7 @@ public class MetadataCreateIndexServiceTests extends ESTestCase { ComponentTemplate ct2 = new ComponentTemplate(ctt2, null, null); IndexTemplateV2 template = new IndexTemplateV2(Collections.singletonList("index"), null, Arrays.asList("ct2", "ct1"), - null, null, null); + null, null, null, null); ClusterState state = ClusterState.builder(ClusterState.EMPTY_STATE) .metadata(Metadata.builder(Metadata.EMPTY_METADATA) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index a7a4660a3a0..76e582291ac 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -308,17 +308,17 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { IndexTemplateV2 firstGlobalIndexTemplate = new IndexTemplateV2(org.elasticsearch.common.collect.List.of("*"), template, org.elasticsearch.common.collect.List.of("foo"), - 1L, null, null); + 1L, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "globalindextemplate1", firstGlobalIndexTemplate); IndexTemplateV2 secondGlobalIndexTemplate = new IndexTemplateV2(org.elasticsearch.common.collect.List.of("*"), template, org.elasticsearch.common.collect.List.of("foo"), - 2L, null, null); + 2L, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "globalindextemplate2", secondGlobalIndexTemplate); IndexTemplateV2 fooPatternIndexTemplate = new IndexTemplateV2(org.elasticsearch.common.collect.List.of("foo-*"), template, org.elasticsearch.common.collect.List.of( - "foo"), 3L, null, null); + "foo"), 3L, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "foopatternindextemplate", fooPatternIndexTemplate); // update the component template to set the index.hidden setting @@ -370,7 +370,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { List patterns = new ArrayList<>(template.indexPatterns()); patterns.add("new-pattern"); template = new IndexTemplateV2(patterns, template.template(), template.composedOf(), template.priority(), template.version(), - template.metadata()); + template.metadata(), null); state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "foo", template); assertNotNull(state.metadata().templatesV2().get("foo")); @@ -407,7 +407,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { .build()) .build(); - IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null); + IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + @@ -444,7 +444,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { waitToCreateComponentTemplate.await(10, TimeUnit.SECONDS); IndexTemplateV2 globalIndexTemplate = new IndexTemplateV2(org.elasticsearch.common.collect.List.of("*"), null, - org.elasticsearch.common.collect.List.of("ct-with-index-hidden-setting"), null, null, null); + org.elasticsearch.common.collect.List.of("ct-with-index-hidden-setting"), null, null, null, null); expectThrows(InvalidIndexTemplateException.class, () -> metadataIndexTemplateService.putIndexTemplateV2("testing", true, "template-referencing-ct-with-hidden-index-setting", @@ -468,7 +468,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { */ public void testPuttingV1StarTemplateGeneratesWarning() throws Exception { final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); - IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null); + IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null, null); ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template); MetadataIndexTemplateService.PutRequest req = new MetadataIndexTemplateService.PutRequest("cause", "v1-template"); @@ -488,7 +488,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { */ public void testPuttingV1NonStarTemplateGeneratesWarning() throws Exception { final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); - IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null); + IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null, null); ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template); MetadataIndexTemplateService.PutRequest req = new MetadataIndexTemplateService.PutRequest("cause", "v1-template"); @@ -517,7 +517,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { .build()) .build(); - IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null); + IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + @@ -557,7 +557,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { .build()) .build(); - IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null); + IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + @@ -581,10 +581,10 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { public void testPuttingOverlappingV2Template() throws Exception { { - IndexTemplateV2 template = new IndexTemplateV2(Arrays.asList("egg*", "baz"), null, null, 1L, null, null); + IndexTemplateV2 template = new IndexTemplateV2(Arrays.asList("egg*", "baz"), null, null, 1L, null, null, null); MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", template); - IndexTemplateV2 newTemplate = new IndexTemplateV2(Arrays.asList("abc", "baz*"), null, null, 1L, null, null); + IndexTemplateV2 newTemplate = new IndexTemplateV2(Arrays.asList("abc", "baz*"), null, null, 1L, null, null, null); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> metadataIndexTemplateService.addIndexTemplateV2(state, false, "foo2", newTemplate)); assertThat(e.getMessage(), equalTo("index template [foo2] has index patterns [abc, baz*] matching patterns from existing " + @@ -593,10 +593,10 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { } { - IndexTemplateV2 template = new IndexTemplateV2(Arrays.asList("egg*", "baz"), null, null, null, null, null); + IndexTemplateV2 template = new IndexTemplateV2(Arrays.asList("egg*", "baz"), null, null, null, null, null, null); MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", template); - IndexTemplateV2 newTemplate = new IndexTemplateV2(Arrays.asList("abc", "baz*"), null, null, 0L, null, null); + IndexTemplateV2 newTemplate = new IndexTemplateV2(Arrays.asList("abc", "baz*"), null, null, 0L, null, null, null); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> metadataIndexTemplateService.addIndexTemplateV2(state, false, "foo2", newTemplate)); assertThat(e.getMessage(), equalTo("index template [foo2] has index patterns [abc, baz*] matching patterns from existing " + @@ -612,9 +612,11 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { ComponentTemplate ct = ComponentTemplateTests.randomInstance(); state = service.addComponentTemplate(state, true, "ct", ct); - IndexTemplateV2 it = new IndexTemplateV2(Collections.singletonList("i*"), null, Collections.singletonList("ct"), null, 1L, null); + IndexTemplateV2 it = + new IndexTemplateV2(Collections.singletonList("i*"), null, Collections.singletonList("ct"), null, 1L, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); - IndexTemplateV2 it2 = new IndexTemplateV2(Collections.singletonList("in*"), null, Collections.singletonList("ct"), 10L, 2L, null); + IndexTemplateV2 it2 = + new IndexTemplateV2(Collections.singletonList("in*"), null, Collections.singletonList("ct"), 10L, 2L, null, null); state = service.addIndexTemplateV2(state, true, "my-template2", it2); String result = MetadataIndexTemplateService.findV2Template(state.metadata(), "index", randomBoolean()); @@ -629,9 +631,11 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { ComponentTemplate ct = ComponentTemplateTests.randomInstance(); state = service.addComponentTemplate(state, true, "ct", ct); - IndexTemplateV2 it = new IndexTemplateV2(Collections.singletonList("i*"), null, Collections.singletonList("ct"), 0L, 1L, null); + IndexTemplateV2 it = + new IndexTemplateV2(Collections.singletonList("i*"), null, Collections.singletonList("ct"), 0L, 1L, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); - IndexTemplateV2 it2 = new IndexTemplateV2(Collections.singletonList("*"), null, Collections.singletonList("ct"), 10L, 2L, null); + IndexTemplateV2 it2 = + new IndexTemplateV2(Collections.singletonList("*"), null, Collections.singletonList("ct"), 10L, 2L, null, null); state = service.addIndexTemplateV2(state, true, "my-template2", it2); String result = MetadataIndexTemplateService.findV2Template(state.metadata(), "index", true); @@ -644,7 +648,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { try { // add an invalid global template that specifies the `index.hidden` setting IndexTemplateV2 invalidGlobalTemplate = new IndexTemplateV2(org.elasticsearch.common.collect.List.of("*"), - templateWithHiddenSetting, org.elasticsearch.common.collect.List.of("ct"), 5L, 1L, null); + templateWithHiddenSetting, org.elasticsearch.common.collect.List.of("ct"), 5L, 1L, null, null); Metadata invalidGlobalTemplateMetadata = Metadata.builder().putCustom(IndexTemplateV2Metadata.TYPE, new IndexTemplateV2Metadata(org.elasticsearch.common.collect.Map.of("invalid_global_template", invalidGlobalTemplate))) .build(); @@ -688,7 +692,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { " }\n" + " }\n" + " }"), null), - Arrays.asList("ct_low", "ct_high"), 0L, 1L, null); + Arrays.asList("ct_low", "ct_high"), 0L, 1L, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); List mappings = MetadataIndexTemplateService.resolveMappings(state, "my-template"); @@ -744,7 +748,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { .put("index.blocks.write", false) .put("index.number_of_shards", 3) .build(), null, null), - Arrays.asList("ct_low", "ct_high"), 0L, 1L, null); + Arrays.asList("ct_low", "ct_high"), 0L, 1L, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), "my-template"); @@ -772,7 +776,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { state = service.addComponentTemplate(state, true, "ct_low", ct2); IndexTemplateV2 it = new IndexTemplateV2(Collections.singletonList("i*"), new Template(null, null, a3), - Arrays.asList("ct_low", "ct_high"), 0L, 1L, null); + Arrays.asList("ct_low", "ct_high"), 0L, 1L, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); List> resolvedAliases = MetadataIndexTemplateService.resolveAliases(state.metadata(), "my-template"); @@ -857,9 +861,9 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { @SuppressWarnings("unchecked") public static void assertTemplatesEqual(IndexTemplateV2 actual, IndexTemplateV2 expected) { IndexTemplateV2 actualNoTemplate = new IndexTemplateV2(actual.indexPatterns(), null, - actual.composedOf(), actual.priority(), actual.version(), actual.metadata()); + actual.composedOf(), actual.priority(), actual.version(), actual.metadata(), actual.getDataStreamTemplate()); IndexTemplateV2 expectedNoTemplate = new IndexTemplateV2(expected.indexPatterns(), null, - expected.composedOf(), expected.priority(), expected.version(), expected.metadata()); + expected.composedOf(), expected.priority(), expected.version(), expected.metadata(), expected.getDataStreamTemplate()); assertThat(actualNoTemplate, equalTo(expectedNoTemplate)); Template actualTemplate = actual.template(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index cdbca8c2cc0..05afb8c4115 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -76,7 +76,8 @@ public class ToAndFromJsonMetadataTests extends ESTestCase { Collections.singletonList("component_template"), 5L, 4L, - Collections.singletonMap("my_meta", Collections.singletonMap("potato", "chicken")))) + Collections.singletonMap("my_meta", Collections.singletonMap("potato", "chicken")), + randomBoolean() ? null : new IndexTemplateV2.DataStreamTemplate("@timestamp"))) .put(IndexMetadata.builder("test12") .settings(settings(Version.CURRENT) .put("setting1", "value1") diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml index 06af6e0aee4..81295f28d4e 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml @@ -2,7 +2,7 @@ "Verify data stream resolvability for xpack apis": - skip: version: " - 7.8.99" - reason: skip untill backported + reason: "data streams only supported in 7.9+" - do: indices.create_data_stream: