From 7dda9934f90b5083031eb60ee2c9ccd103d4222a Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 22 Jun 2020 17:46:38 +0200 Subject: [PATCH] Keep track of timestamp_field mapping as part of a data stream (#58400) Backporting #58096 to 7.x branch. Relates to #53100 * use mapping source direcly instead of using mapper service to extract the relevant mapping details * moved assertion to TimestampField class and added helper method for tests * Improved logic that inserts timestamp field mapping into an mapping. If the timestamp field path consisted out of object fields and if the final mapping did not contain the parent field then an error occurred, because the prior logic assumed that the object field existed. --- .../client/indices/DataStream.java | 22 +-- .../indices/GetDataStreamResponseTests.java | 10 +- .../test/indices.data_stream/10_basic.yml | 21 +- .../30_auto_create_data_stream.yml | 2 +- .../indices.delete/20_backing_indices.yml | 2 +- .../test/indices.rollover/50_data_streams.yml | 2 +- .../elasticsearch/indices/DataStreamIT.java | 179 ++++++++++++++++-- .../snapshots/DataStreamsSnapshotsIT.java | 6 +- .../SharedClusterSnapshotRestoreIT.java | 2 +- .../indices/resolve/ResolveIndexAction.java | 2 +- .../cluster/metadata/DataStream.java | 121 +++++++++++- .../MetadataCreateDataStreamService.java | 36 +++- .../metadata/MetadataCreateIndexService.java | 9 + .../DeleteDataStreamRequestTests.java | 3 +- .../GetDataStreamsRequestTests.java | 9 +- .../indices/resolve/ResolveIndexTests.java | 3 +- .../MetadataRolloverServiceTests.java | 2 +- .../ElasticsearchNodeCommandTests.java | 4 +- .../cluster/metadata/DataStreamTests.java | 79 +++++++- .../IndexNameExpressionResolverTests.java | 21 +- .../MetadataCreateDataStreamServiceTests.java | 4 +- .../MetadataIndexAliasesServiceTests.java | 3 +- .../cluster/metadata/MetadataTests.java | 23 ++- .../metadata/ToAndFromJsonMetadataTests.java | 11 +- .../snapshots/RestoreServiceTests.java | 7 +- .../cluster/DataStreamTestHelper.java | 5 + .../CheckNoDataStreamWriteIndexStepTests.java | 8 +- .../xpack/core/ilm/DeleteStepTests.java | 5 +- ...eplaceDataStreamBackingIndexStepTests.java | 10 +- .../xpack/core/ilm/RolloverStepTests.java | 4 +- .../UpdateRolloverLifecycleDateStepTests.java | 4 +- .../core/ilm/WaitForActiveShardsTests.java | 4 +- .../ilm/WaitForRolloverReadyStepTests.java | 4 +- .../datafeed/DatafeedNodeSelectorTests.java | 3 +- .../authz/IndicesAndAliasesResolverTests.java | 9 +- 35 files changed, 514 insertions(+), 125 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java index d0a08bcedd8..77f896d1f8b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java @@ -20,8 +20,6 @@ package org.elasticsearch.client.indices; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.ToXContentObject; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; @@ -30,12 +28,12 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -public final class DataStream implements ToXContentObject { +public final class DataStream { private final String name; private final String timeStampField; private final List indices; - private long generation; + private final long generation; public DataStream(String name, String timeStampField, List indices, long generation) { this.name = name; @@ -68,14 +66,15 @@ public final class DataStream implements ToXContentObject { @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("data_stream", args -> { + String timeStampField = (String) ((Map) args[1]).get("name"); List indices = ((List>) args[2]).stream().map(m -> m.get("index_name")).collect(Collectors.toList()); - return new DataStream((String) args[0], (String) args[1], indices, (Long) args[3]); + return new DataStream((String) args[0], timeStampField, indices, (Long) args[3]); }); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD); - PARSER.declareString(ConstructingObjectParser.constructorArg(), TIMESTAMP_FIELD_FIELD); + PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> p.map(), TIMESTAMP_FIELD_FIELD); PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> p.mapStrings(), INDICES_FIELD); PARSER.declareLong(ConstructingObjectParser.constructorArg(), GENERATION_FIELD); } @@ -84,17 +83,6 @@ public final class DataStream implements ToXContentObject { return PARSER.parse(parser, null); } - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(NAME_FIELD.getPreferredName(), name); - builder.field(TIMESTAMP_FIELD_FIELD.getPreferredName(), timeStampField); - builder.field(INDICES_FIELD.getPreferredName(), indices); - builder.field(GENERATION_FIELD.getPreferredName(), generation); - builder.endObject(); - return builder; - } - @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java index af936f86621..f86edc49db8 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java @@ -34,6 +34,7 @@ import java.util.List; import java.util.Locale; import java.util.stream.Collectors; +import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName; public class GetDataStreamResponseTests extends AbstractResponseTestCase { @@ -52,12 +53,7 @@ public class GetDataStreamResponseTests extends AbstractResponseTestCase mappings = getIndexResponse.getMappings().get(backingIndex).get("_doc").getSourceAsMap(); + assertThat(ObjectPath.eval("properties.@timestamp2.type", mappings), is("date")); backingIndex = DataStream.getDefaultBackingIndexName("metrics-foo", 1); getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet(); assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue()); assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true)); + mappings = getIndexResponse.getMappings().get(backingIndex).get("_doc").getSourceAsMap(); + assertThat(ObjectPath.eval("properties.@timestamp1.type", mappings), is("date")); int numDocsBar = randomIntBetween(2, 16); indexDocs("metrics-bar", numDocsBar); @@ -152,11 +159,15 @@ public class DataStreamIT extends ESIntegTestCase { getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet(); assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue()); assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true)); + mappings = getIndexResponse.getMappings().get(backingIndex).get("_doc").getSourceAsMap(); + assertThat(ObjectPath.eval("properties.@timestamp1.type", mappings), is("date")); backingIndex = DataStream.getDefaultBackingIndexName("metrics-bar", 2); getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet(); assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue()); assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true)); + mappings = getIndexResponse.getMappings().get(backingIndex).get("_doc").getSourceAsMap(); + assertThat(ObjectPath.eval("properties.@timestamp2.type", mappings), is("date")); int numDocsBar2 = randomIntBetween(2, 16); indexDocs("metrics-bar", numDocsBar2); @@ -186,7 +197,7 @@ public class DataStreamIT extends ESIntegTestCase { } public void testOtherWriteOps() throws Exception { - createIndexTemplate("id", "@timestamp1", "metrics-foobar*"); + putComposableIndexTemplate("id", "@timestamp1", List.of("metrics-foobar*")); String dataStreamName = "metrics-foobar"; CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); client().admin().indices().createDataStream(createDataStreamRequest).get(); @@ -272,7 +283,7 @@ public class DataStreamIT extends ESIntegTestCase { GetDataStreamAction.Response getDataStreamResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); assertThat(getDataStreamResponse.getDataStreams().get(0).getName(), equalTo(dataStreamName)); - assertThat(getDataStreamResponse.getDataStreams().get(0).getTimeStampField(), equalTo("@timestamp")); + assertThat(getDataStreamResponse.getDataStreams().get(0).getTimeStampField().getName(), equalTo("@timestamp")); assertThat(getDataStreamResponse.getDataStreams().get(0).getIndices().size(), equalTo(1)); assertThat(getDataStreamResponse.getDataStreams().get(0).getIndices().get(0).getName(), equalTo(backingIndex)); @@ -352,7 +363,7 @@ public class DataStreamIT extends ESIntegTestCase { } public void testResolvabilityOfDataStreamsInAPIs() throws Exception { - createIndexTemplate("id", "ts", "logs-*"); + putComposableIndexTemplate("id", "ts", List.of("logs-*")); String dataStreamName = "logs-foobar"; CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(dataStreamName); client().admin().indices().createDataStream(request).actionGet(); @@ -413,7 +424,7 @@ public class DataStreamIT extends ESIntegTestCase { } public void testCannotDeleteComposableTemplateUsedByDataStream() throws Exception { - createIndexTemplate("id", "@timestamp1", "metrics-foobar*"); + putComposableIndexTemplate("id", "@timestamp1", List.of("metrics-foobar*")); String dataStreamName = "metrics-foobar-baz"; CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); client().admin().indices().createDataStream(createDataStreamRequest).get(); @@ -436,7 +447,7 @@ public class DataStreamIT extends ESIntegTestCase { } public void testAliasActionsFailOnDataStreams() throws Exception { - createIndexTemplate("id1", "@timestamp1", "metrics-foo*"); + putComposableIndexTemplate("id1", "@timestamp1", List.of("metrics-foo*")); String dataStreamName = "metrics-foo"; CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); client().admin().indices().createDataStream(createDataStreamRequest).get(); @@ -449,7 +460,7 @@ public class DataStreamIT extends ESIntegTestCase { } public void testAliasActionsFailOnDataStreamBackingIndices() throws Exception { - createIndexTemplate("id1", "@timestamp1", "metrics-foo*"); + putComposableIndexTemplate("id1", "@timestamp1", List.of("metrics-foo*")); String dataStreamName = "metrics-foo"; CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); client().admin().indices().createDataStream(createDataStreamRequest).get(); @@ -465,6 +476,133 @@ public class DataStreamIT extends ESIntegTestCase { "support aliases.")); } + public void testChangeTimestampFieldInComposableTemplatePriorToRollOver() throws Exception { + putComposableIndexTemplate("id1", "@timestamp", List.of("logs-foo*")); + + // Index doc that triggers creation of a data stream + IndexRequest indexRequest = new IndexRequest("logs-foobar").source("{}", XContentType.JSON).opType("create"); + IndexResponse indexResponse = client().index(indexRequest).actionGet(); + assertThat(indexResponse.getIndex(), equalTo(DataStream.getDefaultBackingIndexName("logs-foobar", 1))); + assertBackingIndex(DataStream.getDefaultBackingIndexName("logs-foobar", 1), "properties.@timestamp"); + + // Rollover data stream + RolloverResponse rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest("logs-foobar", null)).get(); + assertThat(rolloverResponse.getNewIndex(), equalTo(DataStream.getDefaultBackingIndexName("logs-foobar", 2))); + assertTrue(rolloverResponse.isRolledOver()); + assertBackingIndex(DataStream.getDefaultBackingIndexName("logs-foobar", 2), "properties.@timestamp"); + + // Index another doc into a data stream + indexRequest = new IndexRequest("logs-foobar").source("{}", XContentType.JSON).opType("create"); + indexResponse = client().index(indexRequest).actionGet(); + assertThat(indexResponse.getIndex(), equalTo(DataStream.getDefaultBackingIndexName("logs-foobar", 2))); + + // Change the template to have a different timestamp field + putComposableIndexTemplate("id1", "@timestamp2", List.of("logs-foo*")); + + // Rollover again, eventhough there is no mapping in the template, the timestamp field mapping in data stream + // should be applied in the new backing index + rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest("logs-foobar", null)).get(); + assertThat(rolloverResponse.getNewIndex(), equalTo(DataStream.getDefaultBackingIndexName("logs-foobar", 3))); + assertTrue(rolloverResponse.isRolledOver()); + assertBackingIndex(DataStream.getDefaultBackingIndexName("logs-foobar", 3), "properties.@timestamp"); + + // Index another doc into a data stream + indexRequest = new IndexRequest("logs-foobar").source("{}", XContentType.JSON).opType("create"); + indexResponse = client().index(indexRequest).actionGet(); + assertThat(indexResponse.getIndex(), equalTo(DataStream.getDefaultBackingIndexName("logs-foobar", 3))); + + DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request("logs-foobar"); + client().admin().indices().deleteDataStream(deleteDataStreamRequest).actionGet(); + } + + public void testNestedTimestampField() throws Exception { + String mapping = "{\n" + + " \"properties\": {\n" + + " \"event\": {\n" + + " \"properties\": {\n" + + " \"@timestamp\": {\n" + + " \"type\": \"date\"" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }";; + putComposableIndexTemplate("id1", "event.@timestamp", mapping, List.of("logs-foo*")); + + CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("logs-foobar"); + client().admin().indices().createDataStream(createDataStreamRequest).get(); + GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request("logs-foobar"); + GetDataStreamAction.Response getDataStreamResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); + assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); + assertThat(getDataStreamResponse.getDataStreams().get(0).getName(), equalTo("logs-foobar")); + assertThat(getDataStreamResponse.getDataStreams().get(0).getTimeStampField().getName(), equalTo("event.@timestamp")); + assertThat(getDataStreamResponse.getDataStreams().get(0).getTimeStampField().getFieldMapping(), equalTo(Map.of("type", "date"))); + assertBackingIndex(DataStream.getDefaultBackingIndexName("logs-foobar", 1), "properties.event.properties.@timestamp"); + + // Change the template to have a different timestamp field + putComposableIndexTemplate("id1", "@timestamp2", List.of("logs-foo*")); + + RolloverResponse rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest("logs-foobar", null)).actionGet(); + assertThat(rolloverResponse.getNewIndex(), equalTo(DataStream.getDefaultBackingIndexName("logs-foobar", 2))); + assertTrue(rolloverResponse.isRolledOver()); + assertBackingIndex(DataStream.getDefaultBackingIndexName("logs-foobar", 2), "properties.event.properties.@timestamp"); + + DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request("logs-foobar"); + client().admin().indices().deleteDataStream(deleteDataStreamRequest).actionGet(); + } + + public void testTimestampFieldCustomAttributes() throws Exception { + String mapping = "{\n" + + " \"properties\": {\n" + + " \"@timestamp\": {\n" + + " \"type\": \"date\",\n" + + " \"format\": \"yyyy-MM\",\n" + + " \"meta\": {\n" + + " \"x\": \"y\"\n" + + " },\n" + + " \"store\": true\n" + + " }\n" + + " }\n" + + " }"; + putComposableIndexTemplate("id1", "@timestamp", mapping, List.of("logs-foo*")); + + CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("logs-foobar"); + client().admin().indices().createDataStream(createDataStreamRequest).get(); + GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request("logs-foobar"); + GetDataStreamAction.Response getDataStreamResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); + assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); + assertThat(getDataStreamResponse.getDataStreams().get(0).getName(), equalTo("logs-foobar")); + assertThat(getDataStreamResponse.getDataStreams().get(0).getTimeStampField().getName(), equalTo("@timestamp")); + java.util.Map expectedTimestampMapping = + Map.of("type", "date", "format", "yyyy-MM", "meta", Map.of("x", "y"), "store", true); + assertThat(getDataStreamResponse.getDataStreams().get(0).getTimeStampField().getFieldMapping(), equalTo(expectedTimestampMapping)); + assertBackingIndex(DataStream.getDefaultBackingIndexName("logs-foobar", 1), "properties.@timestamp", expectedTimestampMapping); + + // Change the template to have a different timestamp field + putComposableIndexTemplate("id1", "@timestamp2", List.of("logs-foo*")); + + RolloverResponse rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest("logs-foobar", null)).actionGet(); + assertThat(rolloverResponse.getNewIndex(), equalTo(DataStream.getDefaultBackingIndexName("logs-foobar", 2))); + assertTrue(rolloverResponse.isRolledOver()); + assertBackingIndex(DataStream.getDefaultBackingIndexName("logs-foobar", 2), "properties.@timestamp", expectedTimestampMapping); + + DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request("logs-foobar"); + client().admin().indices().deleteDataStream(deleteDataStreamRequest).actionGet(); + } + + private static void assertBackingIndex(String backingIndex, String timestampFieldPathInMapping) { + assertBackingIndex(backingIndex, timestampFieldPathInMapping, Map.of("type", "date")); + } + + private static void assertBackingIndex(String backingIndex, String timestampFieldPathInMapping, java.util.Map expectedMapping) { + GetIndexResponse getIndexResponse = + client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet(); + assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue()); + assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true)); + java.util.Map mappings = getIndexResponse.getMappings().get(backingIndex).get("_doc").getSourceAsMap(); + assertThat(ObjectPath.eval(timestampFieldPathInMapping, mappings), is(expectedMapping)); + } + private static void verifyResolvability(String dataStream, ActionRequestBuilder requestBuilder, boolean fail) { verifyResolvability(dataStream, requestBuilder, fail, 0); } @@ -524,7 +662,7 @@ public class DataStreamIT extends ESIntegTestCase { SearchResponse searchResponse = client().search(searchRequest).actionGet(); assertThat(searchResponse.getHits().getTotalHits().value, equalTo(expectedNumHits)); - List expectedIndices = new ArrayList<>(); + java.util.List expectedIndices = new ArrayList<>(); for (long k = minGeneration; k <= maxGeneration; k++) { expectedIndices.add(DataStream.getDefaultBackingIndexName(dataStream, k)); } @@ -539,13 +677,22 @@ public class DataStreamIT extends ESIntegTestCase { "] matches a data stream, specify the corresponding concrete indices instead.")); } - public static void createIndexTemplate(String id, String timestampFieldName, String... pattern) throws IOException { + public static void putComposableIndexTemplate(String id, + String timestampFieldName, + java.util.List patterns) throws IOException { + String mapping = MetadataCreateDataStreamServiceTests.generateMapping(timestampFieldName); + putComposableIndexTemplate(id, timestampFieldName, mapping, patterns); + } + + static void putComposableIndexTemplate(String id, + String timestampFieldName, + String mapping, + java.util.List patterns) throws IOException { PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request(id); request.indexTemplate( new ComposableIndexTemplate( - Arrays.asList(pattern), - new Template(null, - new CompressedXContent(MetadataCreateDataStreamServiceTests.generateMapping(timestampFieldName)), null), + patterns, + new Template(null, new CompressedXContent(mapping), null), null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(timestampFieldName)) ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DataStreamsSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DataStreamsSnapshotsIT.java index 942b008427b..e3a35ab5b28 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DataStreamsSnapshotsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DataStreamsSnapshotsIT.java @@ -32,6 +32,7 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.common.collect.List; import org.elasticsearch.indices.DataStreamIT; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; @@ -39,7 +40,6 @@ import org.junit.Before; import java.nio.file.Path; import java.util.Collections; -import java.util.List; import java.util.Map; public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase { @@ -59,7 +59,7 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase { Path location = randomRepoPath(); createRepository(REPO, "fs", location); - DataStreamIT.createIndexTemplate("t1", "@timestamp", "ds", "other-ds"); + DataStreamIT.putComposableIndexTemplate("t1", "@timestamp", List.of("ds", "other-ds")); CreateDataStreamAction.Request request = new CreateDataStreamAction.Request("ds"); AcknowledgedResponse response = client.admin().indices().createDataStream(request).get(); @@ -89,7 +89,7 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase { assertEquals(RestStatus.OK, status); GetSnapshotsResponse snapshot = client.admin().cluster().prepareGetSnapshots(REPO).setSnapshots(SNAPSHOT).get(); - List snap = snapshot.getSnapshots(); + java.util.List snap = snapshot.getSnapshots(); assertEquals(1, snap.size()); assertEquals(Collections.singletonList(DS_BACKING_INDEX_NAME), snap.get(0).indices()); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 92b88c85d49..f92f9dfc3b1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -2427,7 +2427,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas String dataStream = "test-ds"; - DataStreamIT.createIndexTemplate("dst", "@timestamp", dataStream); + DataStreamIT.putComposableIndexTemplate("dst", "@timestamp", org.elasticsearch.common.collect.List.of(dataStream)); logger.info("--> indexing some data"); for (int i = 0; i < 100; i++) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java index d3c6734b594..f88e188de38 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java @@ -718,7 +718,7 @@ public class ResolveIndexAction extends ActionType dataStreams.add(new ResolvedDataStream( dataStream.getName(), backingIndices, - dataStream.getDataStream().getTimeStampField())); + dataStream.getDataStream().getTimeStampField().getName())); break; default: throw new IllegalStateException("unknown index abstraction type: " + ia.getType()); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 82140986d3e..e73f1906666 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -23,29 +23,36 @@ import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Objects; +import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.ALLOWED_TIMESTAMPFIELD_TYPES; +import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.convertFieldPathToMappingPath; + public final class DataStream extends AbstractDiffable implements ToXContentObject { public static final String BACKING_INDEX_PREFIX = ".ds-"; public static final String DATA_STREAMS_METADATA_FIELD = "data-streams"; private final String name; - private final String timeStampField; + private final TimestampField timeStampField; private final List indices; - private long generation; + private final long generation; - public DataStream(String name, String timeStampField, List indices, long generation) { + public DataStream(String name, TimestampField timeStampField, List indices, long generation) { this.name = name; this.timeStampField = timeStampField; this.indices = indices; @@ -54,7 +61,7 @@ public final class DataStream extends AbstractDiffable implements To assert indices.get(indices.size() - 1).getName().equals(getDefaultBackingIndexName(name, generation)); } - public DataStream(String name, String timeStampField, List indices) { + public DataStream(String name, TimestampField timeStampField, List indices) { this(name, timeStampField, indices, indices.size()); } @@ -62,7 +69,7 @@ public final class DataStream extends AbstractDiffable implements To return name; } - public String getTimeStampField() { + public TimestampField getTimeStampField() { return timeStampField; } @@ -141,7 +148,7 @@ public final class DataStream extends AbstractDiffable implements To } public DataStream(StreamInput in) throws IOException { - this(in.readString(), in.readString(), in.readList(Index::new), in.readVLong()); + this(in.readString(), new TimestampField(in), in.readList(Index::new), in.readVLong()); } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -151,7 +158,7 @@ public final class DataStream extends AbstractDiffable implements To @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(name); - out.writeString(timeStampField); + timeStampField.writeTo(out); out.writeList(indices); out.writeVLong(generation); } @@ -163,11 +170,11 @@ public final class DataStream extends AbstractDiffable implements To @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("data_stream", - args -> new DataStream((String) args[0], (String) args[1], (List) args[2], (Long) args[3])); + args -> new DataStream((String) args[0], (TimestampField) args[1], (List) args[2], (Long) args[3])); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD); - PARSER.declareString(ConstructingObjectParser.constructorArg(), TIMESTAMP_FIELD_FIELD); + PARSER.declareObject(ConstructingObjectParser.constructorArg(), TimestampField.PARSER, TIMESTAMP_FIELD_FIELD); PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> Index.fromXContent(p), INDICES_FIELD); PARSER.declareLong(ConstructingObjectParser.constructorArg(), GENERATION_FIELD); } @@ -202,4 +209,100 @@ public final class DataStream extends AbstractDiffable implements To public int hashCode() { return Objects.hash(name, timeStampField, indices, generation); } + + public static final class TimestampField implements Writeable, ToXContentObject { + + static ParseField NAME_FIELD = new ParseField("name"); + static ParseField FIELD_MAPPING_FIELD = new ParseField("mapping"); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "timestamp_field", + args -> new TimestampField((String) args[0], (Map) args[1]) + ); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.mapOrdered(), FIELD_MAPPING_FIELD); + } + + private final String name; + private final Map fieldMapping; + + public TimestampField(String name, Map fieldMapping) { + assert fieldMapping.containsKey("type") : "no type defined for mapping of timestamp_field"; + assert ALLOWED_TIMESTAMPFIELD_TYPES.contains(fieldMapping.get("type")) : + "invalid type defined for mapping of timestamp_field"; + + this.name = name; + this.fieldMapping = fieldMapping; + } + + public TimestampField(StreamInput in) throws IOException { + this.name = in.readString(); + this.fieldMapping = in.readMap(); + } + + /** + * Force fully inserts the timestamp field mapping into the provided mapping. + * Existing mapping definitions for the timestamp field will be completely overwritten. + * Takes into account if the name of the timestamp field is nested. + * + * @param mappings The mapping to update + */ + public void insertTimestampFieldMapping(Map mappings) { + assert mappings.containsKey("_doc"); + + String mappingPath = convertFieldPathToMappingPath(name); + String parentObjectFieldPath = "_doc." + mappingPath.substring(0, mappingPath.lastIndexOf('.')); + String leafFieldName = mappingPath.substring(mappingPath.lastIndexOf('.') + 1); + + Map changes = new HashMap<>(); + Map current = changes; + for (String key : parentObjectFieldPath.split("\\.")) { + Map map = new HashMap<>(); + current.put(key, map); + current = map; + } + current.put(leafFieldName, fieldMapping); + XContentHelper.update(mappings, changes, false); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + out.writeMap(fieldMapping); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(NAME_FIELD.getPreferredName(), name); + builder.field(FIELD_MAPPING_FIELD.getPreferredName(), fieldMapping); + builder.endObject(); + return builder; + } + + public String getName() { + return name; + } + + public Map getFieldMapping() { + return fieldMapping; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TimestampField that = (TimestampField) o; + return name.equals(that.name) && + fieldMapping.equals(that.fieldMapping); + } + + @Override + public int hashCode() { + return Objects.hash(name, fieldMapping); + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 0526927e3f3..a047c32e1bf 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -34,21 +34,24 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.collect.List; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ObjectPath; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.threadpool.ThreadPool; +import java.util.Arrays; import java.util.LinkedHashSet; import java.util.Collections; import java.util.Locale; +import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; public class MetadataCreateDataStreamService { private static final Logger logger = LogManager.getLogger(MetadataCreateDataStreamService.class); - private static final Set ALLOWED_TIMESTAMPFIELD_TYPES = + public static final Set ALLOWED_TIMESTAMPFIELD_TYPES = new LinkedHashSet<>(List.of(DateFieldMapper.CONTENT_TYPE, DateFieldMapper.DATE_NANOS_CONTENT_TYPE)); private final ClusterService clusterService; @@ -146,10 +149,19 @@ public class MetadataCreateDataStreamService { currentState = metadataCreateIndexService.applyCreateIndexRequest(currentState, createIndexRequest, false); IndexMetadata firstBackingIndex = currentState.metadata().index(firstBackingIndexName); assert firstBackingIndex != null; + assert firstBackingIndex.mapping() != null : "no mapping found for backing index [" + firstBackingIndexName + "]"; - Metadata.Builder builder = Metadata.builder(currentState.metadata()).put( - new DataStream(request.name, template.getDataStreamTemplate().getTimestampField(), - Collections.singletonList(firstBackingIndex.getIndex()))); + String fieldName = template.getDataStreamTemplate().getTimestampField(); + Map mapping = firstBackingIndex.mapping().getSourceAsMap(); + Map timeStampFieldMapping = ObjectPath.eval(convertFieldPathToMappingPath(fieldName), mapping); + + DataStream.TimestampField timestampField = new DataStream.TimestampField( + fieldName, + timeStampFieldMapping + ); + DataStream newDataStream = new DataStream(request.name, timestampField, + Collections.singletonList(firstBackingIndex.getIndex())); + Metadata.Builder builder = Metadata.builder(currentState.metadata()).put(newDataStream); logger.info("adding data stream [{}]", request.name); return ClusterState.builder(currentState).metadata(builder).build(); } @@ -179,4 +191,20 @@ public class MetadataCreateDataStreamService { } } + public static String convertFieldPathToMappingPath(String fieldPath) { + // The mapping won't allow such fields, so this is a sanity check: + assert Arrays.stream(fieldPath.split("\\.")).filter(String::isEmpty).count() == 0L || + fieldPath.startsWith(".") || + fieldPath.endsWith(".") : "illegal field path [" + fieldPath + "]"; + + String mappingPath; + if (fieldPath.indexOf('.') == -1) { + mappingPath = "properties." + fieldPath; + } else { + mappingPath = "properties." + fieldPath.replace(".", ".properties."); + } + + return mappingPath; + } + } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index e362403ca2a..67fe7d9611b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -507,6 +507,15 @@ public class MetadataCreateIndexService { final Map> mappings = resolveV2Mappings(sourceMappings, currentState, templateName, xContentRegistry); + if (request.dataStreamName() != null) { + DataStream dataStream = currentState.metadata().dataStreams().get(request.dataStreamName()); + if (dataStream != null) { + @SuppressWarnings("unchecked") + Map _mappings = mappings; // type erasure for java8 generics :( + dataStream.getTimeStampField().insertTimestampFieldMapping(_mappings); + } + } + final Settings aggregatedIndexSettings = aggregateIndexSettings(currentState, request, MetadataIndexTemplateService.resolveSettings(currentState.metadata(), templateName), diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java index ff35019baf8..82ef2774bec 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java @@ -45,6 +45,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Matchers.any; @@ -162,7 +163,7 @@ public class DeleteDataStreamRequestTests extends AbstractWireSerializingTestCas } allIndices.addAll(backingIndices); - DataStream ds = new DataStream(dsTuple.v1(), "@timestamp", + DataStream ds = new DataStream(dsTuple.v1(), createTimestampField("@timestamp"), backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()), dsTuple.v2()); builder.put(ds); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java index a36f849e6df..bfc4d4f54ac 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.test.AbstractWireSerializingTestCase; import java.util.List; +import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -66,7 +67,7 @@ public class GetDataStreamsRequestTests extends AbstractWireSerializingTestCase< final String dataStreamName = "my-data-stream"; IndexMetadata idx = DataStreamTestHelper.createFirstBackingIndex(dataStreamName).build(); DataStream existingDataStream = - new DataStream(dataStreamName, "timestamp", org.elasticsearch.common.collect.List.of(idx.getIndex())); + new DataStream(dataStreamName, createTimestampField("@timestamp"), org.elasticsearch.common.collect.List.of(idx.getIndex())); ClusterState cs = ClusterState.builder(new ClusterName("_name")) .metadata(Metadata.builder().dataStreams(Map.of(dataStreamName, existingDataStream)).build()).build(); GetDataStreamAction.Request req = new GetDataStreamAction.Request(dataStreamName); @@ -80,8 +81,10 @@ public class GetDataStreamsRequestTests extends AbstractWireSerializingTestCase< IndexMetadata idx1 = DataStreamTestHelper.createFirstBackingIndex(dataStreamNames[0]).build(); IndexMetadata idx2 = DataStreamTestHelper.createFirstBackingIndex(dataStreamNames[1]).build(); - DataStream ds1 = new DataStream(dataStreamNames[0], "timestamp", org.elasticsearch.common.collect.List.of(idx1.getIndex())); - DataStream ds2 = new DataStream(dataStreamNames[1], "timestamp", org.elasticsearch.common.collect.List.of(idx2.getIndex())); + DataStream ds1 = new DataStream(dataStreamNames[0], createTimestampField("@timestamp"), + org.elasticsearch.common.collect.List.of(idx1.getIndex())); + DataStream ds2 = new DataStream(dataStreamNames[1], createTimestampField("@timestamp"), + org.elasticsearch.common.collect.List.of(idx2.getIndex())); ClusterState cs = ClusterState.builder(new ClusterName("_name")) .metadata(Metadata.builder().dataStreams( Map.of(dataStreamNames[0], ds1, dataStreamNames[1], ds2)).build()) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java index 53123b72ff6..2889473ad5d 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java @@ -44,6 +44,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; @@ -238,7 +239,7 @@ public class ResolveIndexTests extends ESTestCase { } allIndices.addAll(backingIndices); - DataStream ds = new DataStream(dataStreamName, timestampField, + DataStream ds = new DataStream(dataStreamName, createTimestampField(timestampField), backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList())); builder.put(ds); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index a55b01173a7..e9f14361f44 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -550,7 +550,7 @@ public class MetadataRolloverServiceTests extends ESTestCase { when(allocationService.reroute(any(ClusterState.class), any(String.class))).then(i -> i.getArguments()[0]); DocumentMapper documentMapper = mock(DocumentMapper.class); when(documentMapper.type()).thenReturn("_doc"); - CompressedXContent mapping = new CompressedXContent(generateMapping(dataStream.getTimeStampField())); + CompressedXContent mapping = new CompressedXContent(generateMapping(dataStream.getTimeStampField().getName())); when(documentMapper.mappingSource()).thenReturn(mapping); RoutingFieldMapper routingFieldMapper = mock(RoutingFieldMapper.class); when(routingFieldMapper.required()).thenReturn(false); diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandTests.java index 6dfa6c8241d..5e10c7fb5dd 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandTests.java @@ -42,6 +42,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static org.elasticsearch.cluster.DataStreamTestHelper.createFirstBackingIndex; +import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; @@ -110,7 +111,8 @@ public class ElasticsearchNodeCommandTests extends ESTestCase { for (int i = 0; i < numDataStreams; i++) { String dataStreamName = "name" + 1; IndexMetadata backingIndex = createFirstBackingIndex(dataStreamName).build(); - mdBuilder.put(new DataStream(dataStreamName, "ts", List.of(backingIndex.getIndex()))); + mdBuilder.put(new DataStream(dataStreamName, createTimestampField("@timestamp"), + List.of(backingIndex.getIndex()))); } } mdBuilder.indexGraveyard(graveyard.build()); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 34d1afb26f0..36d20c0ab6b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -18,7 +18,9 @@ */ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.cluster.metadata.DataStream.TimestampField; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.collect.Map; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; @@ -26,9 +28,11 @@ import org.elasticsearch.test.AbstractSerializingTestCase; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Locale; +import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName; import static org.hamcrest.Matchers.equalTo; @@ -48,7 +52,7 @@ public class DataStreamTests extends AbstractSerializingTestCase { long generation = indices.size() + randomLongBetween(1, 128); String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); indices.add(new Index(getDefaultBackingIndexName(dataStreamName, generation), UUIDs.randomBase64UUID(random()))); - return new DataStream(dataStreamName, randomAlphaOfLength(10), indices, generation); + return new DataStream(dataStreamName, createTimestampField(randomAlphaOfLength(10)), indices, generation); } @Override @@ -88,7 +92,7 @@ public class DataStreamTests extends AbstractSerializingTestCase { for (int k = 1; k <= numBackingIndices; k++) { indices.add(new Index(DataStream.getDefaultBackingIndexName(dataStreamName, k), UUIDs.randomBase64UUID(random()))); } - DataStream original = new DataStream(dataStreamName, "@timestamp", indices); + DataStream original = new DataStream(dataStreamName, createTimestampField("@timestamp"), indices); DataStream updated = original.removeBackingIndex(indices.get(indexToRemove - 1)); assertThat(updated.getName(), equalTo(original.getName())); assertThat(updated.getGeneration(), equalTo(original.getGeneration())); @@ -118,7 +122,7 @@ public class DataStreamTests extends AbstractSerializingTestCase { for (int i = 1; i <= numBackingIndices; i++) { indices.add(new Index(DataStream.getDefaultBackingIndexName(dataStreamName, i), UUIDs.randomBase64UUID(random()))); } - DataStream original = new DataStream(dataStreamName, "@timestamp", indices); + DataStream original = new DataStream(dataStreamName, createTimestampField("@timestamp"), indices); Index newBackingIndex = new Index("replacement-index", UUIDs.randomBase64UUID(random())); DataStream updated = original.replaceBackingIndex(indices.get(indexToReplace), newBackingIndex); @@ -143,7 +147,7 @@ public class DataStreamTests extends AbstractSerializingTestCase { for (int i = 1; i <= numBackingIndices; i++) { indices.add(new Index(DataStream.getDefaultBackingIndexName(dataStreamName, i), UUIDs.randomBase64UUID(random()))); } - DataStream original = new DataStream(dataStreamName, "@timestamp", indices); + DataStream original = new DataStream(dataStreamName, createTimestampField("@timestamp"), indices); Index standaloneIndex = new Index("index-foo", UUIDs.randomBase64UUID(random())); Index newBackingIndex = new Index("replacement-index", UUIDs.randomBase64UUID(random())); @@ -159,9 +163,74 @@ public class DataStreamTests extends AbstractSerializingTestCase { for (int i = 1; i <= numBackingIndices; i++) { indices.add(new Index(DataStream.getDefaultBackingIndexName(dataStreamName, i), UUIDs.randomBase64UUID(random()))); } - DataStream original = new DataStream(dataStreamName, "@timestamp", indices); + DataStream original = new DataStream(dataStreamName, createTimestampField("@timestamp"), indices); Index newBackingIndex = new Index("replacement-index", UUIDs.randomBase64UUID(random())); expectThrows(IllegalArgumentException.class, () -> original.replaceBackingIndex(indices.get(writeIndexPosition), newBackingIndex)); } + + public void testInsertTimestampFieldMapping() { + TimestampField timestampField = new TimestampField("@timestamp", Map.of("type", "date", "meta", Map.of("x", "y"))); + + java.util.Map mappings = + Map.of("_doc", Map.of("properties", new HashMap<>(Map.of("my_field", Map.of("type", "keyword"))))); + timestampField.insertTimestampFieldMapping(mappings); + java.util.Map expectedMapping = Map.of("_doc", Map.of("properties", Map.of("my_field", Map.of("type", "keyword"), + "@timestamp", Map.of("type", "date", "meta", Map.of("x", "y"))))); + assertThat(mappings, equalTo(expectedMapping)); + + // ensure that existing @timestamp definitions get overwritten: + mappings = Map.of("_doc", Map.of("properties", new HashMap<>(Map.of("my_field", Map.of("type", "keyword"), + "@timestamp", new HashMap<>(Map.of("type", "keyword")) )))); + timestampField.insertTimestampFieldMapping(mappings); + expectedMapping = Map.of("_doc", Map.of("properties", Map.of("my_field", Map.of("type", "keyword"), "@timestamp", + Map.of("type", "date", "meta", Map.of("x", "y"))))); + assertThat(mappings, equalTo(expectedMapping)); + } + + public void testInsertNestedTimestampFieldMapping() { + TimestampField timestampField = new TimestampField("event.attr.@timestamp", Map.of("type", "date", "meta", Map.of("x", "y"))); + + java.util.Map mappings = Map.of("_doc", Map.of("properties", Map.of("event", Map.of("properties", Map.of("attr", + Map.of("properties", new HashMap<>(Map.of("my_field", Map.of("type", "keyword"))))))))); + timestampField.insertTimestampFieldMapping(mappings); + java.util.Map expectedMapping = + Map.of("_doc", Map.of("properties", Map.of("event", Map.of("properties", Map.of("attr", + Map.of("properties", new HashMap<>(Map.of("my_field", Map.of("type", "keyword"), + "@timestamp", Map.of("type", "date", "meta", Map.of("x", "y")))))))))); + assertThat(mappings, equalTo(expectedMapping)); + + // ensure that existing @timestamp definitions get overwritten: + mappings = Map.of("_doc", Map.of("properties", Map.of("event", Map.of("properties", Map.of("attr", + Map.of("properties", new HashMap<>(Map.of("my_field", Map.of("type", "keyword"), + "@timestamp", new HashMap<>(Map.of("type", "keyword")) )))))))); + timestampField.insertTimestampFieldMapping(mappings); + expectedMapping = Map.of("_doc", Map.of("properties", Map.of("event", Map.of("properties", Map.of("attr", + Map.of("properties", new HashMap<>(Map.of("my_field", Map.of("type", "keyword"), + "@timestamp", Map.of("type", "date", "meta", Map.of("x", "y")))))))))); + assertThat(mappings, equalTo(expectedMapping)); + + // no event and attr parent objects + mappings = Map.of("_doc", Map.of("properties", new HashMap<>())); + timestampField.insertTimestampFieldMapping(mappings); + expectedMapping = Map.of("_doc", Map.of("properties", Map.of("event", Map.of("properties", Map.of("attr", + Map.of("properties", new HashMap<>(Map.of("@timestamp", Map.of("type", "date", "meta", Map.of("x", "y")))))))))); + assertThat(mappings, equalTo(expectedMapping)); + + // no attr parent object + mappings = Map.of("_doc", Map.of("properties", Map.of("event", Map.of("properties", new HashMap<>())))); + timestampField.insertTimestampFieldMapping(mappings); + expectedMapping = Map.of("_doc", Map.of("properties", Map.of("event", Map.of("properties", Map.of("attr", + Map.of("properties", new HashMap<>(Map.of("@timestamp", Map.of("type", "date", "meta", Map.of("x", "y")))))))))); + assertThat(mappings, equalTo(expectedMapping)); + + // Empty attr parent object + mappings = Map.of("_doc", Map.of("properties", Map.of("event", Map.of("properties", + Map.of("attr", Map.of("properties", new HashMap<>())))))); + timestampField.insertTimestampFieldMapping(mappings); + expectedMapping = Map.of("_doc", Map.of("properties", Map.of("event", Map.of("properties", Map.of("attr", + Map.of("properties", new HashMap<>(Map.of("@timestamp", Map.of("type", "date", "meta", Map.of("x", "y")))))))))); + assertThat(mappings, equalTo(expectedMapping)); + } + } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index 8d8be06587b..cf9e408a195 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -50,6 +50,7 @@ import java.util.Set; import java.util.function.Function; import static org.elasticsearch.cluster.DataStreamTestHelper.createBackingIndex; +import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_HIDDEN_SETTING; import static org.elasticsearch.common.util.set.Sets.newHashSet; import static org.hamcrest.Matchers.arrayContaining; @@ -1694,7 +1695,8 @@ public class IndexNameExpressionResolverTests extends ESTestCase { Metadata.Builder mdBuilder = Metadata.builder() .put(backingIndex, false) - .put(new DataStream(dataStreamName, "ts", org.elasticsearch.common.collect.List.of(backingIndex.getIndex()), 1)); + .put(new DataStream(dataStreamName, createTimestampField("ts"), + org.elasticsearch.common.collect.List.of(backingIndex.getIndex()), 1)); ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); { @@ -1804,7 +1806,8 @@ public class IndexNameExpressionResolverTests extends ESTestCase { Metadata.Builder mdBuilder = Metadata.builder() .put(index1, false) .put(index2, false) - .put(new DataStream(dataStreamName, "ts", org.elasticsearch.common.collect.List.of(index1.getIndex(), index2.getIndex()), 2)); + .put(new DataStream(dataStreamName, createTimestampField("ts"), + org.elasticsearch.common.collect.List.of(index1.getIndex(), index2.getIndex()), 2)); ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); { @@ -1886,8 +1889,10 @@ public class IndexNameExpressionResolverTests extends ESTestCase { .put(index2, false) .put(index3, false) .put(index4, false) - .put(new DataStream(dataStream1, "ts", org.elasticsearch.common.collect.List.of(index1.getIndex(), index2.getIndex()))) - .put(new DataStream(dataStream2, "ts", org.elasticsearch.common.collect.List.of(index3.getIndex(), index4.getIndex()))); + .put(new DataStream(dataStream1, createTimestampField("@timestamp"), + org.elasticsearch.common.collect.List.of(index1.getIndex(), index2.getIndex()))) + .put(new DataStream(dataStream2, createTimestampField("@timestamp"), + org.elasticsearch.common.collect.List.of(index3.getIndex(), index4.getIndex()))); ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); { @@ -1933,8 +1938,8 @@ public class IndexNameExpressionResolverTests extends ESTestCase { .put(index1, false) .put(index2, false) .put(justAnIndex, false) - .put(new DataStream(dataStream1, "ts", - org.elasticsearch.common.collect.List.of(index1.getIndex(), index2.getIndex())))).build(); + .put(new DataStream(dataStream1, createTimestampField("@timestamp"), + Arrays.asList(index1.getIndex(), index2.getIndex())))).build(); IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled(); Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "logs-*"); @@ -1967,8 +1972,8 @@ public class IndexNameExpressionResolverTests extends ESTestCase { .put(index3, false) .put(index4, false) .put(justAnIndex, false) - .put(new DataStream(dataStream1, "ts", Arrays.asList(index1.getIndex(), index2.getIndex()))) - .put(new DataStream(dataStream2, "ts", Arrays.asList(index3.getIndex(), index4.getIndex())))).build(); + .put(new DataStream(dataStream1, createTimestampField("ts"), Arrays.asList(index1.getIndex(), index2.getIndex()))) + .put(new DataStream(dataStream2, createTimestampField("ts"), Arrays.asList(index3.getIndex(), index4.getIndex())))).build(); List names = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.lenientExpand(), "log*"); assertEquals(Collections.singletonList(dataStream1), names); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index 0193775d7fa..40a13b2be1c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.util.Collections; import static org.elasticsearch.cluster.DataStreamTestHelper.createFirstBackingIndex; +import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -66,7 +67,8 @@ public class MetadataCreateDataStreamServiceTests extends ESTestCase { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); final String dataStreamName = "my-data-stream"; IndexMetadata idx = createFirstBackingIndex(dataStreamName).build(); - DataStream existingDataStream = new DataStream(dataStreamName, "timestamp", Collections.singletonList(idx.getIndex())); + DataStream existingDataStream = + new DataStream(dataStreamName, createTimestampField("@timestamp"), Collections.singletonList(idx.getIndex())); ClusterState cs = ClusterState.builder(new ClusterName("_name")) .metadata(Metadata.builder().dataStreams(Collections.singletonMap(dataStreamName, existingDataStream)).build()).build(); CreateDataStreamClusterStateUpdateRequest req = diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java index 82d8b522679..01418db55cc 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java @@ -38,6 +38,7 @@ import java.util.List; import java.util.Set; import static java.util.Collections.singletonList; +import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -498,7 +499,7 @@ public class MetadataIndexAliasesServiceTests extends ESTestCase { .metadata( Metadata.builder() .put(indexMetadata, true) - .put(new DataStream(dataStreamName, "@timestamp", singletonList(indexMetadata.getIndex())))) + .put(new DataStream(dataStreamName, createTimestampField("@timestamp"), singletonList(indexMetadata.getIndex())))) .build(); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> service.applyAliasActions(state, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index 399422b5e06..be395e8daec 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -57,6 +57,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.cluster.DataStreamTestHelper.createBackingIndex; import static org.elasticsearch.cluster.DataStreamTestHelper.createFirstBackingIndex; +import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; import static org.elasticsearch.cluster.metadata.Metadata.Builder.validateDataStreams; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -978,7 +979,8 @@ public class MetadataTests extends ESTestCase { .numberOfShards(1) .numberOfReplicas(1) .build(), false) - .put(new DataStream(dataStreamName, "ts", org.elasticsearch.common.collect.List.of(idx.getIndex()))); + .put(new DataStream(dataStreamName, createTimestampField("@timestamp"), + org.elasticsearch.common.collect.List.of(idx.getIndex()))); IllegalStateException e = expectThrows(IllegalStateException.class, b::build); assertThat(e.getMessage(), @@ -993,7 +995,8 @@ public class MetadataTests extends ESTestCase { .build(); Metadata.Builder b = Metadata.builder() .put(idx, false) - .put(new DataStream(dataStreamName, "ts", org.elasticsearch.common.collect.List.of(idx.getIndex()))); + .put(new DataStream(dataStreamName, createTimestampField("@timestamp"), + org.elasticsearch.common.collect.List.of(idx.getIndex()))); IllegalStateException e = expectThrows(IllegalStateException.class, b::build); assertThat(e.getMessage(), @@ -1010,7 +1013,8 @@ public class MetadataTests extends ESTestCase { Metadata.Builder b = Metadata.builder() .put(validIdx, false) .put(invalidIdx, false) - .put(new DataStream(dataStreamName, "ts", org.elasticsearch.common.collect.List.of(validIdx.getIndex()))); + .put(new DataStream(dataStreamName, createTimestampField("@timestamp"), + org.elasticsearch.common.collect.List.of(validIdx.getIndex()))); IllegalStateException e = expectThrows(IllegalStateException.class, b::build); assertThat(e.getMessage(), containsString("data stream [" + dataStreamName + @@ -1025,7 +1029,8 @@ public class MetadataTests extends ESTestCase { .build(); Metadata.Builder b = Metadata.builder() .put(idx, false) - .put(new DataStream(dataStreamName, "ts", org.elasticsearch.common.collect.List.of(idx.getIndex()))); + .put(new DataStream(dataStreamName, createTimestampField("@timestamp"), + org.elasticsearch.common.collect.List.of(idx.getIndex()))); IllegalStateException e = expectThrows(IllegalStateException.class, b::build); assertThat(e.getMessage(), containsString("data stream [" + dataStreamName + @@ -1049,7 +1054,7 @@ public class MetadataTests extends ESTestCase { backingIndices.add(im.getIndex()); } - b.put(new DataStream(dataStreamName, "ts", backingIndices, lastBackingIndexNum)); + b.put(new DataStream(dataStreamName, createTimestampField("ts"), backingIndices, lastBackingIndexNum)); Metadata metadata = b.build(); assertThat(metadata.dataStreams().size(), equalTo(1)); assertThat(metadata.dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); @@ -1067,7 +1072,7 @@ public class MetadataTests extends ESTestCase { indices.add(idx.getIndex()); b.put(idx, true); } - b.put(new DataStream(name, "ts", indices, indices.size())); + b.put(new DataStream(name, createTimestampField("ts"), indices, indices.size())); } Metadata metadata = b.build(); @@ -1132,7 +1137,7 @@ public class MetadataTests extends ESTestCase { } DataStream dataStream = new DataStream( dataStreamName, - "ts", + createTimestampField("ts"), backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()), backingIndices.size() ); @@ -1206,7 +1211,7 @@ public class MetadataTests extends ESTestCase { } DataStream dataStream = new DataStream( dataStreamName, - "ts", + createTimestampField("ts"), backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()), backingIndices.size() ); @@ -1308,7 +1313,7 @@ public class MetadataTests extends ESTestCase { b.put(im, false); backingIndices.add(im.getIndex()); } - b.put(new DataStream(dataStreamName, "ts", backingIndices, lastBackingIndexNum)); + b.put(new DataStream(dataStreamName, createTimestampField("ts"), backingIndices, lastBackingIndexNum)); return new CreateIndexResult(indices, backingIndices, b.build()); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index b524be020ad..6125eaefd98 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -41,6 +41,7 @@ import java.util.HashSet; import java.util.Map; import static org.elasticsearch.cluster.DataStreamTestHelper.createFirstBackingIndex; +import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; import static org.elasticsearch.cluster.metadata.AliasMetadata.newAliasMetadataBuilder; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; import static org.elasticsearch.cluster.metadata.Metadata.CONTEXT_MODE_API; @@ -101,8 +102,10 @@ public class ToAndFromJsonMetadataTests extends ESTestCase { .putAlias(newAliasMetadataBuilder("alias-bar3").routing("routing-bar"))) .put(idx1, false) .put(idx2, false) - .put(new DataStream("data-stream1", "@timestamp", org.elasticsearch.common.collect.List.of(idx1.getIndex()))) - .put(new DataStream("data-stream2", "@timestamp2", org.elasticsearch.common.collect.List.of(idx2.getIndex()))) + .put(new DataStream("data-stream1", createTimestampField("@timestamp"), + org.elasticsearch.common.collect.List.of(idx1.getIndex()))) + .put(new DataStream("data-stream2", createTimestampField("@timestamp2"), + org.elasticsearch.common.collect.List.of(idx2.getIndex()))) .build(); XContentBuilder builder = JsonXContent.contentBuilder(); @@ -152,11 +155,11 @@ public class ToAndFromJsonMetadataTests extends ESTestCase { // data streams assertNotNull(parsedMetadata.dataStreams().get("data-stream1")); assertThat(parsedMetadata.dataStreams().get("data-stream1").getName(), is("data-stream1")); - assertThat(parsedMetadata.dataStreams().get("data-stream1").getTimeStampField(), is("@timestamp")); + assertThat(parsedMetadata.dataStreams().get("data-stream1").getTimeStampField().getName(), is("@timestamp")); assertThat(parsedMetadata.dataStreams().get("data-stream1").getIndices(), contains(idx1.getIndex())); assertNotNull(parsedMetadata.dataStreams().get("data-stream2")); assertThat(parsedMetadata.dataStreams().get("data-stream2").getName(), is("data-stream2")); - assertThat(parsedMetadata.dataStreams().get("data-stream2").getTimeStampField(), is("@timestamp2")); + assertThat(parsedMetadata.dataStreams().get("data-stream2").getTimeStampField().getName(), is("@timestamp2")); assertThat(parsedMetadata.dataStreams().get("data-stream2").getIndices(), contains(idx2.getIndex())); } diff --git a/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java index 5c45c5ea74a..377f82cc28c 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.test.ESTestCase; import java.util.Collections; import java.util.List; +import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -40,7 +41,7 @@ public class RestoreServiceTests extends ESTestCase { String backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1); List indices = Collections.singletonList(new Index(backingIndexName, "uuid")); - DataStream dataStream = new DataStream(dataStreamName, "@timestamp", indices); + DataStream dataStream = new DataStream(dataStreamName, createTimestampField("@timestamp"), indices); Metadata.Builder metadata = mock(Metadata.Builder.class); IndexMetadata indexMetadata = mock(IndexMetadata.class); @@ -63,7 +64,7 @@ public class RestoreServiceTests extends ESTestCase { String renamedBackingIndexName = DataStream.getDefaultBackingIndexName(renamedDataStreamName, 1); List indices = Collections.singletonList(new Index(backingIndexName, "uuid")); - DataStream dataStream = new DataStream(dataStreamName, "@timestamp", indices); + DataStream dataStream = new DataStream(dataStreamName, createTimestampField("@timestamp"), indices); Metadata.Builder metadata = mock(Metadata.Builder.class); IndexMetadata indexMetadata = mock(IndexMetadata.class); @@ -86,7 +87,7 @@ public class RestoreServiceTests extends ESTestCase { String renamedBackingIndexName = DataStream.getDefaultBackingIndexName(renamedDataStreamName, 1); List indices = Collections.singletonList(new Index(backingIndexName, "uuid")); - DataStream dataStream = new DataStream(dataStreamName, "@timestamp", indices); + DataStream dataStream = new DataStream(dataStreamName, createTimestampField("@timestamp"), indices); Metadata.Builder metadata = mock(Metadata.Builder.class); IndexMetadata indexMetadata = mock(IndexMetadata.class); diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java index 3705878c050..bd77f77143f 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java @@ -22,6 +22,7 @@ package org.elasticsearch.cluster; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.collect.Map; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.test.ESTestCase; @@ -51,4 +52,8 @@ public final class DataStreamTestHelper { .numberOfShards(NUMBER_OF_SHARDS) .numberOfReplicas(NUMBER_OF_REPLICAS); } + + public static DataStream.TimestampField createTimestampField(String fieldName) { + return new DataStream.TimestampField(fieldName, Map.of("type", "date")); + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckNoDataStreamWriteIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckNoDataStreamWriteIndexStepTests.java index fbfc58f67f7..c3671297e45 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckNoDataStreamWriteIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckNoDataStreamWriteIndexStepTests.java @@ -12,8 +12,10 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.index.Index; +import java.util.Arrays; import java.util.List; +import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; import static org.elasticsearch.xpack.core.ilm.AbstractStepMasterTimeoutTestCase.emptyClusterState; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -73,8 +75,8 @@ public class CheckNoDataStreamWriteIndexStepTests extends AbstractStepTestCase { @@ -95,7 +96,8 @@ public class UpdateRolloverLifecycleDateStepTests extends AbstractStepTestCase conditionsMet = new SetOnce<>(); Metadata metadata = Metadata.builder().put(indexMetadata, true) - .put(new DataStream(dataStreamName, "timestamp", org.elasticsearch.common.collect.List.of(indexMetadata.getIndex()), 1L)) + .put(new DataStream(dataStreamName, createTimestampField("timestamp"), + org.elasticsearch.common.collect.List.of(indexMetadata.getIndex()), 1L)) .build(); step.evaluateCondition(metadata, indexMetadata.getIndex(), new AsyncWaitStep.Listener() { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java index 50f30e14b7e..cdddb03d095 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java @@ -44,6 +44,7 @@ import java.util.Collections; import java.util.Date; import java.util.List; +import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_UUID_NA_VALUE; import static org.elasticsearch.xpack.ml.action.TransportOpenJobActionTests.addJobTask; @@ -494,7 +495,7 @@ public class DatafeedNodeSelectorTests extends ESTestCase { clusterState = ClusterState.builder(new ClusterName("cluster_name")) .metadata(new Metadata.Builder() - .put(new DataStream(dataStreamName, "@timestamp", Collections.singletonList(index), 1L)) + .put(new DataStream(dataStreamName, createTimestampField("@timestamp"), Collections.singletonList(index), 1L)) .putCustom(PersistentTasksCustomMetadata.TYPE, tasks) .putCustom(MlMetadata.TYPE, mlMetadata) .put(indexMetadata, false)) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java index ddc50d01da4..43d30260779 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java @@ -81,6 +81,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; @@ -167,10 +168,10 @@ public class IndicesAndAliasesResolverTests extends ESTestCase { .put(dataStreamIndex1, true) .put(dataStreamIndex2, true) .put(dataStreamIndex3, true) - .put(new DataStream(dataStreamName, "ts", - org.elasticsearch.common.collect.List.of(dataStreamIndex1.getIndex(), dataStreamIndex2.getIndex()))) - .put(new DataStream(otherDataStreamName, "ts", - org.elasticsearch.common.collect.List.of(dataStreamIndex3.getIndex()))) + .put(new DataStream(dataStreamName, createTimestampField("@timestamp"), + Arrays.asList(dataStreamIndex1.getIndex(), dataStreamIndex2.getIndex()))) + .put(new DataStream(otherDataStreamName, createTimestampField("@timestamp"), + Arrays.asList(dataStreamIndex3.getIndex()))) .put(indexBuilder(securityIndexName).settings(settings)).build(); if (withAlias) {