Default to @timestamp in composable template datastream definition (#59317) (#59516)

This makes the data_stream timestamp field specification optional when
defining a composable template.
When there isn't one specified it will default to `@timestamp`.

(cherry picked from commit 5609353c5d164e15a636c22019c9c17fa98aac30)
Signed-off-by: Andrei Dan <andrei.dan@elastic.co>

client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java

@@ -2037,7 +2037,7 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase {
         CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"@timestamp\":{\"type\":\"date\"}}}");
         Template template = new Template(null, mappings, null);
         ComposableIndexTemplate indexTemplate = new ComposableIndexTemplate(Collections.singletonList(dataStreamName), template,
-            Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
+            Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate());
         PutComposableIndexTemplateRequest putComposableIndexTemplateRequest =
             new PutComposableIndexTemplateRequest().name("ds-template").create(true).indexTemplate(indexTemplate);
         AcknowledgedResponse response = execute(putComposableIndexTemplateRequest,

client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java

@@ -86,7 +86,7 @@ public class GetComposableIndexTemplatesResponseTests extends ESTestCase {
         Long priority = randomBoolean() ? null : randomNonNegativeLong();
         Long version = randomBoolean() ? null : randomNonNegativeLong();
         if (randomBoolean()) {
-            dataStreamTemplate = new ComposableIndexTemplate.DataStreamTemplate("@timestamp");
+            dataStreamTemplate = new ComposableIndexTemplate.DataStreamTemplate();
         }
         return new ComposableIndexTemplate(patterns, randomTemplate(), composedOf, priority, version, meta, dataStreamTemplate);
     }

docs/reference/data-streams/change-mappings-and-settings.asciidoc

@@ -52,9 +52,7 @@ PUT /_ilm/policy/logs_policy
 PUT /_index_template/logs_data_stream
 {
   "index_patterns": [ "logs*" ],
-  "data_stream": {
+  "data_stream": {},
-    "timestamp_field": "@timestamp"
-  },
   "template": {
     "mappings": {
       "properties": {
@@ -69,9 +67,7 @@ PUT /_index_template/logs_data_stream
 PUT /_index_template/new_logs_data_stream
 {
   "index_patterns": [ "new_logs*" ],
-  "data_stream": {
+  "data_stream": {},
-    "timestamp_field": "@timestamp"
-  },
   "template": {
     "mappings": {
       "properties": {
@@ -124,9 +120,7 @@ for a new field, `message`, to the template.
 PUT /_index_template/logs_data_stream
 {
   "index_patterns": [ "logs*" ],
-  "data_stream": {
+  "data_stream": {},
-    "timestamp_field": "@timestamp"
-  },
   "template": {
     "mappings": {
       "properties": {
@@ -193,9 +187,7 @@ mapping parameter to `true`.
 PUT /_index_template/logs_data_stream
 {
   "index_patterns": [ "logs*" ],
-  "data_stream": {
+  "data_stream": {},
-    "timestamp_field": "@timestamp"
-  },
   "template": {
     "mappings": {
       "properties": {
@@ -279,9 +271,7 @@ template's `index.refresh_interval` index setting to `30s` (30 seconds).
 PUT /_index_template/logs_data_stream
 {
   "index_patterns": [ "logs*" ],
-  "data_stream": {
+  "data_stream": {},
-    "timestamp_field": "@timestamp"
-  },
   "template": {
     "mappings": {
       "properties": {
@@ -345,9 +335,7 @@ The following <<indices-templates,put index template API>> requests adds new
 PUT /_index_template/logs_data_stream
 {
   "index_patterns": [ "logs*" ],
-  "data_stream": {
+  "data_stream": {},
-    "timestamp_field": "@timestamp"
-  },
   "template": {
     "mappings": {
       "properties": {
@@ -460,9 +448,7 @@ uses the `logs_data_stream` template as its basis, with the following changes:
 PUT /_index_template/new_logs_data_stream
 {
   "index_patterns": [ "new_logs*" ],
-  "data_stream": {
+  "data_stream": {},
-    "timestamp_field": "@timestamp"
-  },
   "template": {
     "mappings": {
       "properties": {

docs/reference/data-streams/set-up-a-data-stream.asciidoc

@@ -134,9 +134,7 @@ this pattern.
 ----
 ====
 
-* A `data_stream` definition containing the `timestamp_field` property.
+* A `data_stream` object with an empty body (`{ }`).
-  This timestamp field must be included in every document indexed to the data
-  stream.
 
 The template can also contain:
 
@@ -176,9 +174,7 @@ configures the `logs_data_stream` template.
 PUT /_index_template/logs_data_stream
 {
   "index_patterns": [ "logs*" ],
-  "data_stream": {
+  "data_stream": {},
-    "timestamp_field": "@timestamp"
-  },
   "template": {
     "settings": {
       "index.lifecycle.name": "logs_policy"

docs/reference/data-streams/use-a-data-stream.asciidoc

@@ -19,8 +19,15 @@ the following:
 PUT /_index_template/logs_data_stream
 {
   "index_patterns": [ "logs*" ],
-  "data_stream": {
+  "data_stream": {},
-    "timestamp_field": "@timestamp"
+  "template": {
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        }
+      }
+    }
   }
 }
 

docs/reference/ilm/ilm-tutorial.asciidoc

@@ -121,19 +121,17 @@ with the options you specify.
 PUT _index_template/timeseries_template
 {
   "index_patterns": ["timeseries"],                   <1>
-  "data_stream": {
+  "data_stream": {},
-    "timestamp_field": "@timestamp"                   <2>
-  },
   "template": {
     "settings": {
       "number_of_shards": 1,
       "number_of_replicas": 1,
-      "index.lifecycle.name": "timeseries_policy"     <3>
+      "index.lifecycle.name": "timeseries_policy"     <2>
     },
     "mappings": {
       "properties": {
         "@timestamp": {
-          "type": "date"                              <4>
+          "type": "date"                              <3>
         }
       }
     }
@@ -143,11 +141,9 @@ PUT _index_template/timeseries_template
 // TEST[continued]
 
 <1> Apply the template when a document is indexed into the `timeseries` target.
-<2> Identifies the timestamp field for the data source. This field must be present
+<2> The name of the {ilm-init} policy used to manage the data stream.
-in all documents indexed into the `timeseries` data stream.
+<3> A <<date,`date`>> or <<date_nanos,`date_nanos`>> field mapping for the
-<3> The name of the {ilm-init} policy used to manage the data stream.
+"@timestamp" field the data stream will use.
-<4> A <<date,`date`>> or <<date_nanos,`date_nanos`>> field mapping for the
-timestamp field specified in the `timestamp_field` property
 ====
 
 [discrete]

docs/reference/indices/create-data-stream.asciidoc

@@ -25,9 +25,7 @@ PUT /_index_template/template
       }
     }
   },
-  "data_stream": {
+  "data_stream": {}
-    "timestamp_field": "@timestamp"
-  }
 }
 ----
 ////

docs/reference/indices/delete-data-stream.asciidoc

@@ -23,9 +23,7 @@ PUT /_index_template/template
       }
     }
   },
-  "data_stream": {
+  "data_stream": {}
-    "timestamp_field": "@timestamp"
-  }
 }
 
 PUT /_data_stream/my-data-stream

docs/reference/indices/get-data-stream.asciidoc

@@ -35,9 +35,7 @@ PUT /_ilm/policy/my-lifecycle-policy
 PUT /_index_template/my-index-template
 {
   "index_patterns": [ "my-data-stream*" ],
-  "data_stream": {
+  "data_stream": {},
-    "timestamp_field": "@timestamp"
-  },
   "template": {
     "mappings": {
       "properties": {

docs/reference/indices/index-templates.asciidoc

@@ -493,19 +493,10 @@ PUT /_index_template/template_1
       }
     }
   },
-  "data_stream": {
+  "data_stream": {}
-    "timestamp_field": "@timestamp"
-  }
 }
 --------------------------------------------------
 
-Required properties of a data stream definition:
-
-`timestamp_field`::
-(Required, string) The name of the timestamp field. This field must be present
-in all documents indexed into the data stream and must be of type
-<<date, `date`>> or <<date_nanos, `date_nanos`>>.
-
 [[multiple-component-templates]]
 ===== Composing multiple component templates
 

docs/reference/indices/resolve.asciidoc

@@ -30,9 +30,7 @@ PUT /my-index/_alias/f-alias
 PUT /_index_template/foo_data_stream
 {
   "index_patterns": [ "foo" ],
-  "data_stream": {
+  "data_stream": {},
-    "timestamp_field": "@timestamp"
-  },
   "template": {
     "mappings": {
       "properties": {

docs/reference/indices/rollover-index.asciidoc

@@ -241,9 +241,7 @@ PUT _index_template/template
       }
     }
   },
-  "data_stream": {
+  "data_stream": {}
-    "timestamp_field": "@timestamp"
-  }
 }
 -----------------------------------
 // TEST

rest-api-spec/src/main/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml

@@ -1,8 +1,8 @@
 ---
 setup:
   - skip:
-      version: "7.8.99 - "
+      version: "7.9.99 - "
-      reason: "resolve index api only supported in 7.9+"
+      reason: "enable in 7.9+ when backported"
       features: allowed_warnings
 
   - do:

server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java

@@ -102,7 +102,7 @@ public class ComposableTemplateIT extends ESIntegTestCase {
         Exception expectedException = expectThrows(Exception.class, () -> ComposableIndexTemplate.parse(parser));
 
         ComposableIndexTemplate template = new ComposableIndexTemplate(List.of("logs-*-*"), null, null, null, null,
-            null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
+            null, new ComposableIndexTemplate.DataStreamTemplate());
         Exception e = expectThrows(IllegalArgumentException.class, () -> client().execute(PutComposableIndexTemplateAction.INSTANCE,
             new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(template)).actionGet());
         Exception actualException = (Exception) e.getCause();

server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java

@@ -29,6 +29,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -249,31 +250,20 @@ public class ComposableIndexTemplate extends AbstractDiffable<ComposableIndexTem
 
     public static class DataStreamTemplate implements Writeable, ToXContentObject {
 
-        private static final ConstructingObjectParser<DataStreamTemplate, Void> PARSER = new ConstructingObjectParser<>(
+        private static final ObjectParser<DataStreamTemplate, Void> PARSER = new ObjectParser<>(
             "data_stream_template",
-            args -> new DataStreamTemplate((String) args[0])
+            DataStreamTemplate::new
         );
 
-        static {
+        public DataStreamTemplate() {
-            PARSER.declareString(ConstructingObjectParser.constructorArg(), DataStream.TIMESTAMP_FIELD_FIELD);
-        }
-
-        private final String timestampField;
-
-        public DataStreamTemplate(String timestampField) {
-            if (FIXED_TIMESTAMP_FIELD.equals(timestampField) == false) {
-                throw new IllegalArgumentException("unexpected timestamp field [" + timestampField + "]");
-            }
-
-            this.timestampField = timestampField;
         }
 
         public String getTimestampField() {
-            return timestampField;
+            return FIXED_TIMESTAMP_FIELD;
         }
 
-        DataStreamTemplate(StreamInput in) throws IOException {
+        DataStreamTemplate(StreamInput in) {
-            this(in.readString());
+            this();
         }
 
         /**
@@ -281,18 +271,16 @@ public class ComposableIndexTemplate extends AbstractDiffable<ComposableIndexTem
          */
         public Map<String, Object> getDataSteamMappingSnippet() {
             return singletonMap(MapperService.SINGLE_MAPPING_NAME, singletonMap("_data_stream_timestamp",
-                singletonMap("path", timestampField)));
+                singletonMap("path", FIXED_TIMESTAMP_FIELD)));
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeString(timestampField);
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
-            builder.field(DataStream.TIMESTAMP_FIELD_FIELD.getPreferredName(), timestampField);
             builder.endObject();
             return builder;
         }
@@ -300,14 +288,12 @@ public class ComposableIndexTemplate extends AbstractDiffable<ComposableIndexTem
         @Override
         public boolean equals(Object o) {
             if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
+            return o != null && getClass() == o.getClass();
-            DataStreamTemplate that = (DataStreamTemplate) o;
-            return timestampField.equals(that.timestampField);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(timestampField);
+            return DataStreamTemplate.class.hashCode();
         }
     }
 }

server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java

@@ -36,7 +36,7 @@ public class AutoCreateActionTests extends ESTestCase {
         Metadata metadata;
         {
             Metadata.Builder mdBuilder = new Metadata.Builder();
-            DataStreamTemplate dataStreamTemplate = new DataStreamTemplate("@timestamp");
+            DataStreamTemplate dataStreamTemplate = new DataStreamTemplate();
             mdBuilder.put("1", new ComposableIndexTemplate(Collections.singletonList("legacy-logs-*"), null, null, 10L, null, null, null));
             mdBuilder.put("2", new ComposableIndexTemplate(Collections.singletonList("logs-*"),
                 null, null, 20L, null, null, dataStreamTemplate));

server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java

@@ -48,8 +48,8 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -539,7 +539,7 @@ public class MetadataRolloverServiceTests extends ESTestCase {
     public void testRolloverClusterStateForDataStream() throws Exception {
         final DataStream dataStream = DataStreamTests.randomInstance();
         ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStream.getName() + "*"),
-            null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
+            null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate());
         Metadata.Builder builder = Metadata.builder();
         builder.put("template", template);
         for (Index index : dataStream.getIndices()) {
@@ -638,7 +638,7 @@ public class MetadataRolloverServiceTests extends ESTestCase {
             sourceIndexName = dataStream.getIndices().get(dataStream.getIndices().size() - 1).getName();
             defaultRolloverIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
             ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStream.getName() + "*"),
-                null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
+                null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate());
             builder.put("template", template);
             for (Index index : dataStream.getIndices()) {
                 builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index));

server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java

@@ -148,7 +148,7 @@ public class ComposableIndexTemplateTests extends AbstractDiffableSerializationT
         if (randomBoolean()) {
             return null;
         } else {
-            return new ComposableIndexTemplate.DataStreamTemplate("@timestamp");
+            return new ComposableIndexTemplate.DataStreamTemplate();
         }
     }
 

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java

@@ -46,7 +46,7 @@ public class MetadataCreateDataStreamServiceTests extends ESTestCase {
         final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService();
         final String dataStreamName = "my-data-stream";
         ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStreamName + "*"),
-            null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
+            null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate());
         ClusterState cs = ClusterState.builder(new ClusterName("_name"))
             .metadata(Metadata.builder().put("template", template).build())
             .build();
@@ -140,7 +140,7 @@ public class MetadataCreateDataStreamServiceTests extends ESTestCase {
     public static ClusterState createDataStream(final String dataStreamName) throws Exception {
         final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService();
         ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStreamName + "*"),
-            null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
+            null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate());
         ClusterState cs = ClusterState.builder(new ClusterName("_name"))
             .metadata(Metadata.builder().put("template", template).build())
             .build();

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java

@@ -847,7 +847,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
                         "    }\n" +
                         "  }"), null),
                 org.elasticsearch.common.collect.List.of("ct1"), 0L, 1L, null,
-                new ComposableIndexTemplate.DataStreamTemplate(DEFAULT_TIMESTAMP_FIELD));
+                new ComposableIndexTemplate.DataStreamTemplate());
             state = service.addIndexTemplateV2(state, true, "logs-data-stream-template", it);
 
             List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "logs-data-stream-template",
@@ -985,7 +985,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
             state = service.addComponentTemplate(state, true, "ct1", ct1);
             ComposableIndexTemplate it = new ComposableIndexTemplate(org.elasticsearch.common.collect.List.of("logs*"), null,
                 org.elasticsearch.common.collect.List.of("ct1"), 0L, 1L, null,
-                new ComposableIndexTemplate.DataStreamTemplate(DEFAULT_TIMESTAMP_FIELD));
+                new ComposableIndexTemplate.DataStreamTemplate());
             state = service.addIndexTemplateV2(state, true, "logs-template", it);
 
             List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "logs-template",
@@ -1030,7 +1030,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
                 "    }"), null);
             ComposableIndexTemplate it = new ComposableIndexTemplate(
                 org.elasticsearch.common.collect.List.of("timeseries*"), template, null, 0L, 1L, null,
-                new ComposableIndexTemplate.DataStreamTemplate(DEFAULT_TIMESTAMP_FIELD));
+                new ComposableIndexTemplate.DataStreamTemplate());
             state = service.addIndexTemplateV2(state, true, "timeseries-template", it);
 
             List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template",
@@ -1352,7 +1352,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
             .build();
 
         ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList("logs-*-*"),
-            null, null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
+            null, null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate());
 
         state = service.addIndexTemplateV2(state, false, "logs", template);
 
@@ -1396,7 +1396,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
         // Change the pattern to one that doesn't match the data stream
         e = expectThrows(IllegalArgumentException.class, () -> {
             ComposableIndexTemplate newTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-postgres-*"), null,
-                null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
+                null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate());
             service.addIndexTemplateV2(stateWithDS, false, "logs", newTemplate);
         });
 
@@ -1406,7 +1406,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
 
         // Add an additional template that matches our data stream at a lower priority
         ComposableIndexTemplate mysqlTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-mysql-*"), null,
-            null, 50L, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
+            null, 50L, null, null, new ComposableIndexTemplate.DataStreamTemplate());
         ClusterState stateWithDSAndTemplate = service.addIndexTemplateV2(stateWithDS, false, "logs-mysql", mysqlTemplate);
 
         // We should be able to replace the "logs" template, because we have the "logs-mysql" template that can handle the data stream

server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java

@@ -78,7 +78,7 @@ public class ToAndFromJsonMetadataTests extends ESTestCase {
                     5L,
                     4L,
                     Collections.singletonMap("my_meta", Collections.singletonMap("potato", "chicken")),
-                    randomBoolean() ? null : new ComposableIndexTemplate.DataStreamTemplate("@timestamp")))
+                    randomBoolean() ? null : new ComposableIndexTemplate.DataStreamTemplate()))
                 .put(IndexMetadata.builder("test12")
                         .settings(settings(Version.CURRENT)
                                 .put("setting1", "value1")

x-pack/docs/en/security/authorization/alias-privileges.asciidoc

@@ -43,9 +43,7 @@ backing indices, the user can retrieve a document directly from `.ds-logs-000002
 PUT /_index_template/logs_data_stream
 {
   "index_patterns": [ "logs*" ],
-  "data_stream": {
+  "data_stream": {},
-    "timestamp_field": "@timestamp"
-  },
   "template": {
     "mappings": {
       "properties": {

x-pack/plugin/core/src/main/resources/logs-template.json

@@ -1,9 +1,7 @@
 {
   "index_patterns": ["logs-*-*"],
   "priority": 100,
-  "data_stream": {
+  "data_stream": {},
-    "timestamp_field": "@timestamp"
-  },
   "composed_of": [
     "logs-mappings",
     "logs-settings"

x-pack/plugin/core/src/main/resources/metrics-template.json

@@ -1,9 +1,7 @@
 {
   "index_patterns": ["metrics-*-*"],
   "priority": 100,
-  "data_stream": {
+  "data_stream": {},
-    "timestamp_field": "@timestamp"
-  },
   "composed_of": [
     "metrics-mappings",
     "metrics-settings"

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/100_delete_by_query.yml

@@ -12,8 +12,7 @@
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/10_basic.yml

@@ -3,7 +3,6 @@ setup:
       features: allowed_warnings
       version: " - 7.9.99"
       reason: "enable in 7.9+ when backported"
-
   - do:
       allowed_warnings:
         - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
@@ -14,8 +13,8 @@ setup:
           template:
             settings:
               index.number_of_replicas: 0
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
+
   - do:
       allowed_warnings:
         - "index template [my-template2] has index patterns [simple-data-stream2] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation"
@@ -28,8 +27,7 @@ setup:
               properties:
                 '@timestamp':
                   type: date_nanos
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
 ---
 "Create data stream":
@@ -243,8 +241,8 @@ setup:
         name: generic_logs_template
         body:
           index_patterns: logs-*
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
+
 
   - do:
       index:
@@ -299,8 +297,7 @@ setup:
         name: generic_logs_template
         body:
           index_patterns: logs-*
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       catch: bad_request

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/110_update_by_query.yml

@@ -12,8 +12,7 @@
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/20_unsupported_apis.yml

@@ -12,8 +12,7 @@
         name: my-template
         body:
           index_patterns: [logs-*]
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:
@@ -61,8 +60,7 @@
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:
@@ -101,7 +99,7 @@
 "Prohibit shrink on data stream's write index":
   - skip:
       version: " - 7.9.99"
-      reason: "data streams only supported in 7.9+"
+      reason: "enable in 7.9+ when backported"
       features: allowed_warnings
 
   - do:
@@ -111,8 +109,7 @@
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:
@@ -149,8 +146,7 @@
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:
@@ -171,7 +167,7 @@
 "Prohibit split on data stream's write index":
   - skip:
       version: " - 7.9.99"
-      reason: "enable in 7.9+ when backported"
+      reason: "data streams only supported in 7.9+"
       features: allowed_warnings
 
   - do:
@@ -181,8 +177,7 @@
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/30_auto_create_data_stream.yml

@@ -12,8 +12,7 @@
         name: generic_logs_template
         body:
           index_patterns: logs-*
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
           template:
             settings:
               number_of_shards:   1

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/40_supported_apis.yml

@@ -4,7 +4,6 @@ setup:
       features: allowed_warnings
       version: " - 7.9.99"
       reason: "enable in 7.9+ when backported"
-
   - do:
       allowed_warnings:
         - "index template [logs_template] has index patterns [logs-foobar] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs_template] will take precedence during new index creation"
@@ -12,8 +11,7 @@ setup:
         name: logs_template
         body:
           index_patterns: logs-foobar
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:
@@ -41,8 +39,8 @@ teardown:
 ---
 "Verify get mapping api":
   - skip:
-      version: " - 7.8.99"
+      version: " - 7.9.99"
-      reason: "data streams only supported in 7.9+"
+      reason: "enable in 7.9+ when backported"
 
   - do:
       indices.get_mapping:
@@ -53,8 +51,9 @@ teardown:
 ---
 "Verify shard stores api":
   - skip:
-      version: " - 7.8.99"
+      version: " - 7.9.99"
-      reason: "data streams only supported in 7.9+"
+      reason: "change to 7.8.99 after backport"
+      features: allowed_warnings
 
   - do:
       allowed_warnings:
@@ -67,8 +66,7 @@ teardown:
             settings:
               number_of_shards: "1"
               number_of_replicas: "0"
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:
@@ -115,8 +113,7 @@ teardown:
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:
@@ -148,8 +145,7 @@ teardown:
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:
@@ -182,8 +178,7 @@ teardown:
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/50_delete_backing_indices.yml

@@ -10,8 +10,7 @@ setup:
         name: my-template
         body:
           index_patterns: [simple-*]
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
 ---
 "Delete backing index on data stream":
@@ -69,7 +68,7 @@ setup:
 "Attempt to delete write index on data stream is rejected":
   - skip:
       version: " - 7.9.99"
-      reason:  "enable in 7.9+ when backported"
+      reason:  "enable in 7.9+ when backported9+"
 
   - do:
       indices.create_data_stream:

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/60_get_backing_indices.yml

@@ -12,8 +12,7 @@
         name: my-template
         body:
           index_patterns: [data-*]
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/70_rollover_data_streams.yml

@@ -12,8 +12,7 @@
         name: my-template
         body:
           index_patterns: [data-*]
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/80_resolve_index_data_streams.yml

@@ -12,8 +12,7 @@ setup:
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       allowed_warnings:
@@ -22,13 +21,7 @@ setup:
         name: my-template2
         body:
           index_patterns: [simple-data-stream2]
-          template:
+          data_stream: {}
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
-          data_stream:
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/90_reindex.yml

@@ -12,8 +12,7 @@ setup:
         name: generic_logs_template
         body:
           index_patterns: logs-*
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
 ---
 teardown:

x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java

@@ -294,7 +294,7 @@ public class DataStreamIT extends ESIntegTestCase {
                 null,
                 null,
                 null,
-                new ComposableIndexTemplate.DataStreamTemplate("@timestamp")
+                new ComposableIndexTemplate.DataStreamTemplate()
             )
         );
         client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet();
@@ -376,7 +376,7 @@ public class DataStreamIT extends ESIntegTestCase {
                 null,
                 null,
                 null,
-                new ComposableIndexTemplate.DataStreamTemplate("@timestamp")
+                new ComposableIndexTemplate.DataStreamTemplate()
             )
         );
 
@@ -837,7 +837,7 @@ public class DataStreamIT extends ESIntegTestCase {
                 null,
                 null,
                 null,
-                new ComposableIndexTemplate.DataStreamTemplate("@timestamp")
+                new ComposableIndexTemplate.DataStreamTemplate()
             )
         );
         client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet();
@@ -1005,7 +1005,7 @@ public class DataStreamIT extends ESIntegTestCase {
                 null,
                 null,
                 null,
-                new ComposableIndexTemplate.DataStreamTemplate("@timestamp")
+                new ComposableIndexTemplate.DataStreamTemplate()
             )
         );
         client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet();

x-pack/plugin/eql/qa/rest/src/test/resources/rest-api-spec/test/eql/20_data_streams.yml

@@ -17,8 +17,7 @@
               properties:
                 '@timestamp':
                   type: date
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:

x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java

@@ -125,7 +125,7 @@ public final class TimeSeriesRestDriver {
         StringEntity templateJSON = new StringEntity(
             String.format(Locale.ROOT, "{\n" +
                 "  \"index_patterns\": \"%s\",\n" +
-                "  \"data_stream\": { \"timestamp_field\": \"@timestamp\" },\n" +
+                "  \"data_stream\": {},\n" +
                 "  \"template\": %s\n" +
                 "}", indexPattern, Strings.toString(builder)),
             ContentType.APPLICATION_JSON);

x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java

@@ -729,7 +729,7 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase {
             "    }";
         if (isDatastream) {
             try {
-                createDataStreamAndTemplate(index, "@timestamp", mapping);
+                createDataStreamAndTemplate(index, mapping);
             } catch (IOException ex) {
                 throw new ElasticsearchException(ex);
             }

x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java

@@ -114,7 +114,7 @@ public class DatafeedJobsIT extends MlNativeAutodetectIntegTestCase {
             "        }" +
             "      }\n" +
             "    }";
-        createDataStreamAndTemplate("datafeed_data_stream", "@timestamp", mapping);
+        createDataStreamAndTemplate("datafeed_data_stream", mapping);
         long numDocs = randomIntBetween(32, 2048);
         long now = System.currentTimeMillis();
         long oneWeekAgo = now - 604800000;

x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java

@@ -275,7 +275,7 @@ abstract class MlNativeIntegTestCase extends ESIntegTestCase {
         }
     }
 
-    protected static void createDataStreamAndTemplate(String dataStreamName, String timeField, String mapping) throws IOException {
+    protected static void createDataStreamAndTemplate(String dataStreamName, String mapping) throws IOException {
         client().execute(PutComposableIndexTemplateAction.INSTANCE,
             new PutComposableIndexTemplateAction.Request(dataStreamName + "_template")
                 .indexTemplate(new ComposableIndexTemplate(Collections.singletonList(dataStreamName),
@@ -284,7 +284,7 @@ abstract class MlNativeIntegTestCase extends ESIntegTestCase {
                     null,
                     null,
                     null,
-                    new ComposableIndexTemplate.DataStreamTemplate(timeField))))
+                    new ComposableIndexTemplate.DataStreamTemplate())))
             .actionGet();
         client().execute(CreateDataStreamAction.INSTANCE, new CreateDataStreamAction.Request(dataStreamName)).actionGet();
     }

x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java

@@ -487,7 +487,7 @@ public class RegressionIT extends MlNativeDataFrameAnalyticsIntegTestCase {
             "    }";
         if (dataStream) {
             try {
-                createDataStreamAndTemplate(sourceIndex, "@timestamp", mapping);
+                createDataStreamAndTemplate(sourceIndex, mapping);
             } catch (IOException ex) {
                 throw new ElasticsearchException(ex);
             }

x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml

@@ -1,8 +1,8 @@
 ---
 "Verify data stream resolvability for xpack apis":
   - skip:
-      version: " - 7.8.99"
+      version: " - 7.9.99"
-      reason: "data streams only supported in 7.9+"
+      reason: "enable in 7.9+ when backported"
       features: allowed_warnings
 
   - do:
@@ -17,8 +17,7 @@
               properties:
                 '@timestamp':
                   type: date
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:
@@ -54,8 +53,8 @@
 ---
 "Verify data stream resolvability in ilm explain api":
   - skip:
-      version: " - 7.8.99"
+      version: " - 7.9.99"
-      reason: "data streams only supported in 7.9+"
+      reason: "enable in 7.9+ when backported"
       features: allowed_warnings
 
   - do:
@@ -88,8 +87,7 @@
         name: generic_logs_template
         body:
           index_patterns: logs-*
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
           template:
             settings:
               index.lifecycle.name: "my_lifecycle"
@@ -119,8 +117,8 @@
 ---
 "Verify data stream resolvability rollup":
   - skip:
-      version: " - 7.8.99"
+      version: " - 7.9.99"
-      reason: "data streams only supported in 7.9+"
+      reason: "enable in 7.9+ when backported"
       features: allowed_warnings
 
   - do:
@@ -135,8 +133,7 @@
               properties:
                 '@timestamp':
                   type: date
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:
@@ -196,8 +193,7 @@
                   type: date
             settings:
               index.lifecycle.name: "my_moveable_timeseries_lifecycle"
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:
@@ -261,8 +257,7 @@
               properties:
                 '@timestamp':
                   type: date
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:
@@ -318,8 +313,7 @@
               properties:
                 '@timestamp':
                   type: date
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:
@@ -358,8 +352,7 @@
               properties:
                 '@timestamp':
                   type: date
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:
@@ -433,8 +426,7 @@
                   type: keyword
                 price:
                   type: integer
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:

x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/10_basic.yml

@@ -136,8 +136,8 @@
 ---
 "Cannot freeze write index for data stream":
   - skip:
-      version: " - 7.8.99"
+      version: " - 7.9.99"
-      reason: "data streams only supported in 7.9+"
+      reason: "enable in 7.9+ when backported"
       features: allowed_warnings
 
   - do:
@@ -152,8 +152,7 @@
               properties:
                 '@timestamp':
                   type: date
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:

x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/50_data_streams.yml

@@ -61,8 +61,7 @@ setup:
               properties:
                 '@timestamp':
                   type: date
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
 ---
 teardown:

x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java

@@ -121,7 +121,6 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
                     "{\n" +
                         "  \"index_patterns\": [ \"" + indexName + "\" ],\n" +
                         "  \"data_stream\": {\n" +
-                        "    \"timestamp_field\": \"@timestamp\"\n" +
                         "  },\n" +
                         "  \"template\": \n" + Strings.toString(builder) +
                         "}"

x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml

@@ -68,8 +68,8 @@ setup:
               properties:
                 '@timestamp':
                   type: date
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
+
   - do:
       allowed_warnings:
         - "index template [my-template2] has index patterns [simple-data-stream2] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation"
@@ -82,8 +82,7 @@ setup:
               properties:
                 '@timestamp':
                   type: date
-          data_stream:
+          data_stream: {}
-            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java

@@ -33,7 +33,6 @@ public class DataStreamsUpgradeIT extends AbstractUpgradeTestCase {
                 "        }\n" +
                 "      },\n" +
                 "      \"data_stream\":{\n" +
-                "        \"timestamp_field\":\"@timestamp\"" +
                 "      }\n" +
                 "    }";
             Request request = new Request("PUT", "/_index_template/1");