honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Default to @timestamp in composable template datastream definition (#59317) (#59516) 

This makes the data_stream timestamp field specification optional when
defining a composable template.
When there isn't one specified it will default to `@timestamp`.

(cherry picked from commit 5609353c5d164e15a636c22019c9c17fa98aac30)
Signed-off-by: Andrei Dan <andrei.dan@elastic.co>
This commit is contained in:
Andrei Dan 2020-07-14 12:36:54 +01:00 committed by GitHub
parent 2e71d0aa91
commit 7dcdaeae49
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
48 changed files with 127 additions and 222 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
View File

@ -2037,7 +2037,7 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase {
CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"@timestamp\":{\"type\":\"date\"}}}");
Template template = new Template(null, mappings, null);
ComposableIndexTemplate indexTemplate = new ComposableIndexTemplate(Collections.singletonList(dataStreamName), template,
Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate());
PutComposableIndexTemplateRequest putComposableIndexTemplateRequest =
new PutComposableIndexTemplateRequest().name("ds-template").create(true).indexTemplate(indexTemplate);
AcknowledgedResponse response = execute(putComposableIndexTemplateRequest,

2
client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java
View File

@ -86,7 +86,7 @@ public class GetComposableIndexTemplatesResponseTests extends ESTestCase {
Long priority = randomBoolean() ? null : randomNonNegativeLong();
Long version = randomBoolean() ? null : randomNonNegativeLong();
if (randomBoolean()) {
dataStreamTemplate = new ComposableIndexTemplate.DataStreamTemplate("@timestamp");
dataStreamTemplate = new ComposableIndexTemplate.DataStreamTemplate();
}
return new ComposableIndexTemplate(patterns, randomTemplate(), composedOf, priority, version, meta, dataStreamTemplate);
}

28
docs/reference/data-streams/change-mappings-and-settings.asciidoc
View File

@ -52,9 +52,7 @@ PUT /_ilm/policy/logs_policy
PUT /_index_template/logs_data_stream
{
"index_patterns": [ "logs*" ],
"data_stream": {
"timestamp_field": "@timestamp"
},
"data_stream": {},
"template": {
"mappings": {
"properties": {
@ -69,9 +67,7 @@ PUT /_index_template/logs_data_stream
PUT /_index_template/new_logs_data_stream
{
"index_patterns": [ "new_logs*" ],
"data_stream": {
"timestamp_field": "@timestamp"
},
"data_stream": {},
"template": {
"mappings": {
"properties": {
@ -124,9 +120,7 @@ for a new field, `message`, to the template.
PUT /_index_template/logs_data_stream
{
"index_patterns": [ "logs*" ],
"data_stream": {
"timestamp_field": "@timestamp"
},
"data_stream": {},
"template": {
"mappings": {
"properties": {
@ -193,9 +187,7 @@ mapping parameter to `true`.
PUT /_index_template/logs_data_stream
{
"index_patterns": [ "logs*" ],
"data_stream": {
"timestamp_field": "@timestamp"
},
"data_stream": {},
"template": {
"mappings": {
"properties": {
@ -279,9 +271,7 @@ template's `index.refresh_interval` index setting to `30s` (30 seconds).
PUT /_index_template/logs_data_stream
{
"index_patterns": [ "logs*" ],
"data_stream": {
"timestamp_field": "@timestamp"
},
"data_stream": {},
"template": {
"mappings": {
"properties": {
@ -345,9 +335,7 @@ The following <<indices-templates,put index template API>> requests adds new
PUT /_index_template/logs_data_stream
{
"index_patterns": [ "logs*" ],
"data_stream": {
"timestamp_field": "@timestamp"
},
"data_stream": {},
"template": {
"mappings": {
"properties": {
@ -460,9 +448,7 @@ uses the `logs_data_stream` template as its basis, with the following changes:
PUT /_index_template/new_logs_data_stream
{
"index_patterns": [ "new_logs*" ],
"data_stream": {
"timestamp_field": "@timestamp"
},
"data_stream": {},
"template": {
"mappings": {
"properties": {

8
docs/reference/data-streams/set-up-a-data-stream.asciidoc
View File

@ -134,9 +134,7 @@ this pattern.
----
====
* A `data_stream` definition containing the `timestamp_field` property.
This timestamp field must be included in every document indexed to the data
stream.
* A `data_stream` object with an empty body (`{ }`).
The template can also contain:
@ -176,9 +174,7 @@ configures the `logs_data_stream` template.
PUT /_index_template/logs_data_stream
{
"index_patterns": [ "logs*" ],
"data_stream": {
"timestamp_field": "@timestamp"
},
"data_stream": {},
"template": {
"settings": {
"index.lifecycle.name": "logs_policy"

11
docs/reference/data-streams/use-a-data-stream.asciidoc
View File

@ -19,8 +19,15 @@ the following:
PUT /_index_template/logs_data_stream
{
"index_patterns": [ "logs*" ],
"data_stream": {
"timestamp_field": "@timestamp"
"data_stream": {},
"template": {
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
}
}
}
}
}

16
docs/reference/ilm/ilm-tutorial.asciidoc
View File

@ -121,19 +121,17 @@ with the options you specify.
PUT _index_template/timeseries_template
{
"index_patterns": ["timeseries"], <1>
"data_stream": {
"timestamp_field": "@timestamp" <2>
},
"data_stream": {},
"template": {
"settings": {
"number_of_shards": 1,
"number_of_replicas": 1,
"index.lifecycle.name": "timeseries_policy" <3>
"index.lifecycle.name": "timeseries_policy" <2>
},
"mappings": {
"properties": {
"@timestamp": {
"type": "date" <4>
"type": "date" <3>
}
}
}
@ -143,11 +141,9 @@ PUT _index_template/timeseries_template
// TEST[continued]
<1> Apply the template when a document is indexed into the `timeseries` target.
<2> Identifies the timestamp field for the data source. This field must be present
in all documents indexed into the `timeseries` data stream.
<3> The name of the {ilm-init} policy used to manage the data stream.
<4> A <<date,`date`>> or <<date_nanos,`date_nanos`>> field mapping for the
timestamp field specified in the `timestamp_field` property
<2> The name of the {ilm-init} policy used to manage the data stream.
<3> A <<date,`date`>> or <<date_nanos,`date_nanos`>> field mapping for the
"@timestamp" field the data stream will use.
====
[discrete]

4
docs/reference/indices/create-data-stream.asciidoc
View File

@ -25,9 +25,7 @@ PUT /_index_template/template
}
}
},
"data_stream": {
"timestamp_field": "@timestamp"
}
"data_stream": {}
}
----
////

4
docs/reference/indices/delete-data-stream.asciidoc
View File

@ -23,9 +23,7 @@ PUT /_index_template/template
}
}
},
"data_stream": {
"timestamp_field": "@timestamp"
}
"data_stream": {}
}
PUT /_data_stream/my-data-stream

4
docs/reference/indices/get-data-stream.asciidoc
View File

@ -35,9 +35,7 @@ PUT /_ilm/policy/my-lifecycle-policy
PUT /_index_template/my-index-template
{
"index_patterns": [ "my-data-stream*" ],
"data_stream": {
"timestamp_field": "@timestamp"
},
"data_stream": {},
"template": {
"mappings": {
"properties": {

11
docs/reference/indices/index-templates.asciidoc
View File

@ -493,19 +493,10 @@ PUT /_index_template/template_1
}
}
},
"data_stream": {
"timestamp_field": "@timestamp"
}
"data_stream": {}
}
--------------------------------------------------
Required properties of a data stream definition:
`timestamp_field`::
(Required, string) The name of the timestamp field. This field must be present
in all documents indexed into the data stream and must be of type
<<date, `date`>> or <<date_nanos, `date_nanos`>>.
[[multiple-component-templates]]
===== Composing multiple component templates

4
docs/reference/indices/resolve.asciidoc
View File

@ -30,9 +30,7 @@ PUT /my-index/_alias/f-alias
PUT /_index_template/foo_data_stream
{
"index_patterns": [ "foo" ],
"data_stream": {
"timestamp_field": "@timestamp"
},
"data_stream": {},
"template": {
"mappings": {
"properties": {

4
docs/reference/indices/rollover-index.asciidoc
View File

@ -241,9 +241,7 @@ PUT _index_template/template
}
}
},
"data_stream": {
"timestamp_field": "@timestamp"
}
"data_stream": {}
}
-----------------------------------
// TEST

4
rest-api-spec/src/main/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml
View File

@ -1,8 +1,8 @@
---
setup:
- skip:
version: "7.8.99 - "
reason: "resolve index api only supported in 7.9+"
version: "7.9.99 - "
reason: "enable in 7.9+ when backported"
features: allowed_warnings
- do:

2
server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java
View File

@ -102,7 +102,7 @@ public class ComposableTemplateIT extends ESIntegTestCase {
Exception expectedException = expectThrows(Exception.class, () -> ComposableIndexTemplate.parse(parser));
ComposableIndexTemplate template = new ComposableIndexTemplate(List.of("logs-*-*"), null, null, null, null,
null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
null, new ComposableIndexTemplate.DataStreamTemplate());
Exception e = expectThrows(IllegalArgumentException.class, () -> client().execute(PutComposableIndexTemplateAction.INSTANCE,
new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(template)).actionGet());
Exception actualException = (Exception) e.getCause();

34
server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java
View File

@ -29,6 +29,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
@ -249,31 +250,20 @@ public class ComposableIndexTemplate extends AbstractDiffable<ComposableIndexTem
public static class DataStreamTemplate implements Writeable, ToXContentObject {
private static final ConstructingObjectParser<DataStreamTemplate, Void> PARSER = new ConstructingObjectParser<>(
private static final ObjectParser<DataStreamTemplate, Void> PARSER = new ObjectParser<>(
"data_stream_template",
args -> new DataStreamTemplate((String) args[0])
DataStreamTemplate::new
);
static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), DataStream.TIMESTAMP_FIELD_FIELD);
}
private final String timestampField;
public DataStreamTemplate(String timestampField) {
if (FIXED_TIMESTAMP_FIELD.equals(timestampField) == false) {
throw new IllegalArgumentException("unexpected timestamp field [" + timestampField + "]");
}
this.timestampField = timestampField;
public DataStreamTemplate() {
}
public String getTimestampField() {
return timestampField;
return FIXED_TIMESTAMP_FIELD;
}
DataStreamTemplate(StreamInput in) throws IOException {
this(in.readString());
DataStreamTemplate(StreamInput in) {
this();
}
/**
@ -281,18 +271,16 @@ public class ComposableIndexTemplate extends AbstractDiffable<ComposableIndexTem
*/
public Map<String, Object> getDataSteamMappingSnippet() {
return singletonMap(MapperService.SINGLE_MAPPING_NAME, singletonMap("_data_stream_timestamp",
singletonMap("path", timestampField)));
singletonMap("path", FIXED_TIMESTAMP_FIELD)));
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(timestampField);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(DataStream.TIMESTAMP_FIELD_FIELD.getPreferredName(), timestampField);
builder.endObject();
return builder;
}
@ -300,14 +288,12 @@ public class ComposableIndexTemplate extends AbstractDiffable<ComposableIndexTem
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DataStreamTemplate that = (DataStreamTemplate) o;
return timestampField.equals(that.timestampField);
return o != null && getClass() == o.getClass();
}
@Override
public int hashCode() {
return Objects.hash(timestampField);
return DataStreamTemplate.class.hashCode();
}
}
}

2
server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java
View File

@ -36,7 +36,7 @@ public class AutoCreateActionTests extends ESTestCase {
Metadata metadata;
{
Metadata.Builder mdBuilder = new Metadata.Builder();
DataStreamTemplate dataStreamTemplate = new DataStreamTemplate("@timestamp");
DataStreamTemplate dataStreamTemplate = new DataStreamTemplate();
mdBuilder.put("1", new ComposableIndexTemplate(Collections.singletonList("legacy-logs-*"), null, null, 10L, null, null, null));
mdBuilder.put("2", new ComposableIndexTemplate(Collections.singletonList("logs-*"),
null, null, 20L, null, null, dataStreamTemplate));

6
server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
View File

@ -48,8 +48,8 @@ import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.json.JsonXContent;
@ -539,7 +539,7 @@ public class MetadataRolloverServiceTests extends ESTestCase {
public void testRolloverClusterStateForDataStream() throws Exception {
final DataStream dataStream = DataStreamTests.randomInstance();
ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStream.getName() + "*"),
null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate());
Metadata.Builder builder = Metadata.builder();
builder.put("template", template);
for (Index index : dataStream.getIndices()) {
@ -638,7 +638,7 @@ public class MetadataRolloverServiceTests extends ESTestCase {
sourceIndexName = dataStream.getIndices().get(dataStream.getIndices().size() - 1).getName();
defaultRolloverIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStream.getName() + "*"),
null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate());
builder.put("template", template);
for (Index index : dataStream.getIndices()) {
builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index));

2
server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java
View File

@ -148,7 +148,7 @@ public class ComposableIndexTemplateTests extends AbstractDiffableSerializationT
if (randomBoolean()) {
return null;
} else {
return new ComposableIndexTemplate.DataStreamTemplate("@timestamp");
return new ComposableIndexTemplate.DataStreamTemplate();
}
}

4
server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java
View File

@ -46,7 +46,7 @@ public class MetadataCreateDataStreamServiceTests extends ESTestCase {
final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService();
final String dataStreamName = "my-data-stream";
ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStreamName + "*"),
null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate());
ClusterState cs = ClusterState.builder(new ClusterName("_name"))
.metadata(Metadata.builder().put("template", template).build())
.build();
@ -140,7 +140,7 @@ public class MetadataCreateDataStreamServiceTests extends ESTestCase {
public static ClusterState createDataStream(final String dataStreamName) throws Exception {
final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService();
ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStreamName + "*"),
null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate());
ClusterState cs = ClusterState.builder(new ClusterName("_name"))
.metadata(Metadata.builder().put("template", template).build())
.build();

12
server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
View File

@ -847,7 +847,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
" }\n" +
" }"), null),
org.elasticsearch.common.collect.List.of("ct1"), 0L, 1L, null,
new ComposableIndexTemplate.DataStreamTemplate(DEFAULT_TIMESTAMP_FIELD));
new ComposableIndexTemplate.DataStreamTemplate());
state = service.addIndexTemplateV2(state, true, "logs-data-stream-template", it);
List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "logs-data-stream-template",
@ -985,7 +985,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
state = service.addComponentTemplate(state, true, "ct1", ct1);
ComposableIndexTemplate it = new ComposableIndexTemplate(org.elasticsearch.common.collect.List.of("logs*"), null,
org.elasticsearch.common.collect.List.of("ct1"), 0L, 1L, null,
new ComposableIndexTemplate.DataStreamTemplate(DEFAULT_TIMESTAMP_FIELD));
new ComposableIndexTemplate.DataStreamTemplate());
state = service.addIndexTemplateV2(state, true, "logs-template", it);
List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "logs-template",
@ -1030,7 +1030,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
" }"), null);
ComposableIndexTemplate it = new ComposableIndexTemplate(
org.elasticsearch.common.collect.List.of("timeseries*"), template, null, 0L, 1L, null,
new ComposableIndexTemplate.DataStreamTemplate(DEFAULT_TIMESTAMP_FIELD));
new ComposableIndexTemplate.DataStreamTemplate());
state = service.addIndexTemplateV2(state, true, "timeseries-template", it);
List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template",
@ -1352,7 +1352,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
.build();
ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList("logs-*-*"),
null, null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
null, null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate());
state = service.addIndexTemplateV2(state, false, "logs", template);
@ -1396,7 +1396,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
// Change the pattern to one that doesn't match the data stream
e = expectThrows(IllegalArgumentException.class, () -> {
ComposableIndexTemplate newTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-postgres-*"), null,
null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate());
service.addIndexTemplateV2(stateWithDS, false, "logs", newTemplate);
});
@ -1406,7 +1406,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
// Add an additional template that matches our data stream at a lower priority
ComposableIndexTemplate mysqlTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-mysql-*"), null,
null, 50L, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
null, 50L, null, null, new ComposableIndexTemplate.DataStreamTemplate());
ClusterState stateWithDSAndTemplate = service.addIndexTemplateV2(stateWithDS, false, "logs-mysql", mysqlTemplate);
// We should be able to replace the "logs" template, because we have the "logs-mysql" template that can handle the data stream

2
server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java
View File

@ -78,7 +78,7 @@ public class ToAndFromJsonMetadataTests extends ESTestCase {
5L,
4L,
Collections.singletonMap("my_meta", Collections.singletonMap("potato", "chicken")),
randomBoolean() ? null : new ComposableIndexTemplate.DataStreamTemplate("@timestamp")))
randomBoolean() ? null : new ComposableIndexTemplate.DataStreamTemplate()))
.put(IndexMetadata.builder("test12")
.settings(settings(Version.CURRENT)
.put("setting1", "value1")

4
x-pack/docs/en/security/authorization/alias-privileges.asciidoc
View File

@ -43,9 +43,7 @@ backing indices, the user can retrieve a document directly from `.ds-logs-000002
PUT /_index_template/logs_data_stream
{
"index_patterns": [ "logs*" ],
"data_stream": {
"timestamp_field": "@timestamp"
},
"data_stream": {},
"template": {
"mappings": {
"properties": {

4
x-pack/plugin/core/src/main/resources/logs-template.json
View File

@ -1,9 +1,7 @@
{
"index_patterns": ["logs-*-*"],
"priority": 100,
"data_stream": {
"timestamp_field": "@timestamp"
},
"data_stream": {},
"composed_of": [
"logs-mappings",
"logs-settings"

4
x-pack/plugin/core/src/main/resources/metrics-template.json
View File

@ -1,9 +1,7 @@
{
"index_patterns": ["metrics-*-*"],
"priority": 100,
"data_stream": {
"timestamp_field": "@timestamp"
},
"data_stream": {},
"composed_of": [
"metrics-mappings",
"metrics-settings"

3
x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/100_delete_by_query.yml
View File

@ -12,8 +12,7 @@
name: my-template1
body:
index_patterns: [simple-data-stream1]
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:

15
x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/10_basic.yml
View File

@ -3,7 +3,6 @@ setup:
features: allowed_warnings
version: " - 7.9.99"
reason: "enable in 7.9+ when backported"
- do:
allowed_warnings:
- "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
@ -14,8 +13,8 @@ setup:
template:
settings:
index.number_of_replicas: 0
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
allowed_warnings:
- "index template [my-template2] has index patterns [simple-data-stream2] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation"
@ -28,8 +27,7 @@ setup:
properties:
'@timestamp':
type: date_nanos
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
---
"Create data stream":
@ -243,8 +241,8 @@ setup:
name: generic_logs_template
body:
index_patterns: logs-*
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
index:
@ -299,8 +297,7 @@ setup:
name: generic_logs_template
body:
index_patterns: logs-*
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
catch: bad_request

3
x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/110_update_by_query.yml
View File

@ -12,8 +12,7 @@
name: my-template1
body:
index_patterns: [simple-data-stream1]
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:

19
x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/20_unsupported_apis.yml
View File

@ -12,8 +12,7 @@
name: my-template
body:
index_patterns: [logs-*]
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:
@ -61,8 +60,7 @@
name: my-template1
body:
index_patterns: [simple-data-stream1]
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:
@ -101,7 +99,7 @@
"Prohibit shrink on data stream's write index":
- skip:
version: " - 7.9.99"
reason: "data streams only supported in 7.9+"
reason: "enable in 7.9+ when backported"
features: allowed_warnings
- do:
@ -111,8 +109,7 @@
name: my-template1
body:
index_patterns: [simple-data-stream1]
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:
@ -149,8 +146,7 @@
name: my-template1
body:
index_patterns: [simple-data-stream1]
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:
@ -171,7 +167,7 @@
"Prohibit split on data stream's write index":
- skip:
version: " - 7.9.99"
reason: "enable in 7.9+ when backported"
reason: "data streams only supported in 7.9+"
features: allowed_warnings
- do:
@ -181,8 +177,7 @@
name: my-template1
body:
index_patterns: [simple-data-stream1]
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:

3
x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/30_auto_create_data_stream.yml
View File

@ -12,8 +12,7 @@
name: generic_logs_template
body:
index_patterns: logs-*
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
template:
settings:
number_of_shards: 1

25
x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/40_supported_apis.yml
View File

@ -4,7 +4,6 @@ setup:
features: allowed_warnings
version: " - 7.9.99"
reason: "enable in 7.9+ when backported"
- do:
allowed_warnings:
- "index template [logs_template] has index patterns [logs-foobar] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs_template] will take precedence during new index creation"
@ -12,8 +11,7 @@ setup:
name: logs_template
body:
index_patterns: logs-foobar
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:
@ -41,8 +39,8 @@ teardown:
---
"Verify get mapping api":
- skip:
version: " - 7.8.99"
reason: "data streams only supported in 7.9+"
version: " - 7.9.99"
reason: "enable in 7.9+ when backported"
- do:
indices.get_mapping:
@ -53,8 +51,9 @@ teardown:
---
"Verify shard stores api":
- skip:
version: " - 7.8.99"
reason: "data streams only supported in 7.9+"
version: " - 7.9.99"
reason: "change to 7.8.99 after backport"
features: allowed_warnings
- do:
allowed_warnings:
@ -67,8 +66,7 @@ teardown:
settings:
number_of_shards: "1"
number_of_replicas: "0"
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:
@ -115,8 +113,7 @@ teardown:
name: my-template1
body:
index_patterns: [simple-data-stream1]
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:
@ -148,8 +145,7 @@ teardown:
name: my-template1
body:
index_patterns: [simple-data-stream1]
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:
@ -182,8 +178,7 @@ teardown:
name: my-template1
body:
index_patterns: [simple-data-stream1]
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:

5
x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/50_delete_backing_indices.yml
View File

@ -10,8 +10,7 @@ setup:
name: my-template
body:
index_patterns: [simple-*]
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
---
"Delete backing index on data stream":
@ -69,7 +68,7 @@ setup:
"Attempt to delete write index on data stream is rejected":
- skip:
version: " - 7.9.99"
reason: "enable in 7.9+ when backported"
reason: "enable in 7.9+ when backported9+"
- do:
indices.create_data_stream:

3
x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/60_get_backing_indices.yml
View File

@ -12,8 +12,7 @@
name: my-template
body:
index_patterns: [data-*]
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:

3
x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/70_rollover_data_streams.yml
View File

@ -12,8 +12,7 @@
name: my-template
body:
index_patterns: [data-*]
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:

11
x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/80_resolve_index_data_streams.yml
View File

@ -12,8 +12,7 @@ setup:
name: my-template1
body:
index_patterns: [simple-data-stream1]
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
allowed_warnings:
@ -22,13 +21,7 @@ setup:
name: my-template2
body:
index_patterns: [simple-data-stream2]
template:
mappings:
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:

3
x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/90_reindex.yml
View File

@ -12,8 +12,7 @@ setup:
name: generic_logs_template
body:
index_patterns: logs-*
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
---
teardown:

8
x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java
View File

@ -294,7 +294,7 @@ public class DataStreamIT extends ESIntegTestCase {
null,
null,
null,
new ComposableIndexTemplate.DataStreamTemplate("@timestamp")
new ComposableIndexTemplate.DataStreamTemplate()
)
);
client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet();
@ -376,7 +376,7 @@ public class DataStreamIT extends ESIntegTestCase {
null,
null,
null,
new ComposableIndexTemplate.DataStreamTemplate("@timestamp")
new ComposableIndexTemplate.DataStreamTemplate()
)
);
@ -837,7 +837,7 @@ public class DataStreamIT extends ESIntegTestCase {
null,
null,
null,
new ComposableIndexTemplate.DataStreamTemplate("@timestamp")
new ComposableIndexTemplate.DataStreamTemplate()
)
);
client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet();
@ -1005,7 +1005,7 @@ public class DataStreamIT extends ESIntegTestCase {
null,
null,
null,
new ComposableIndexTemplate.DataStreamTemplate("@timestamp")
new ComposableIndexTemplate.DataStreamTemplate()
)
);
client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet();

3
x-pack/plugin/eql/qa/rest/src/test/resources/rest-api-spec/test/eql/20_data_streams.yml
View File

@ -17,8 +17,7 @@
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:

2
x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java
View File

@ -125,7 +125,7 @@ public final class TimeSeriesRestDriver {
StringEntity templateJSON = new StringEntity(
String.format(Locale.ROOT, "{\n" +
" \"index_patterns\": \"%s\",\n" +
" \"data_stream\": { \"timestamp_field\": \"@timestamp\" },\n" +
" \"data_stream\": {},\n" +
" \"template\": %s\n" +
"}", indexPattern, Strings.toString(builder)),
ContentType.APPLICATION_JSON);

2
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
View File

@ -729,7 +729,7 @@ public class ClassificationIT extends MlNativeDataFrameAnalyticsIntegTestCase {
" }";
if (isDatastream) {
try {
createDataStreamAndTemplate(index, "@timestamp", mapping);
createDataStreamAndTemplate(index, mapping);
} catch (IOException ex) {
throw new ElasticsearchException(ex);
}

2
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java
View File

@ -114,7 +114,7 @@ public class DatafeedJobsIT extends MlNativeAutodetectIntegTestCase {
" }" +
" }\n" +
" }";
createDataStreamAndTemplate("datafeed_data_stream", "@timestamp", mapping);
createDataStreamAndTemplate("datafeed_data_stream", mapping);
long numDocs = randomIntBetween(32, 2048);
long now = System.currentTimeMillis();
long oneWeekAgo = now - 604800000;

4
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java
View File

@ -275,7 +275,7 @@ abstract class MlNativeIntegTestCase extends ESIntegTestCase {
}
}
protected static void createDataStreamAndTemplate(String dataStreamName, String timeField, String mapping) throws IOException {
protected static void createDataStreamAndTemplate(String dataStreamName, String mapping) throws IOException {
client().execute(PutComposableIndexTemplateAction.INSTANCE,
new PutComposableIndexTemplateAction.Request(dataStreamName + "_template")
.indexTemplate(new ComposableIndexTemplate(Collections.singletonList(dataStreamName),
@ -284,7 +284,7 @@ abstract class MlNativeIntegTestCase extends ESIntegTestCase {
null,
null,
null,
new ComposableIndexTemplate.DataStreamTemplate(timeField))))
new ComposableIndexTemplate.DataStreamTemplate())))
.actionGet();
client().execute(CreateDataStreamAction.INSTANCE, new CreateDataStreamAction.Request(dataStreamName)).actionGet();
}

2
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
View File

@ -487,7 +487,7 @@ public class RegressionIT extends MlNativeDataFrameAnalyticsIntegTestCase {
" }";
if (dataStream) {
try {
createDataStreamAndTemplate(sourceIndex, "@timestamp", mapping);
createDataStreamAndTemplate(sourceIndex, mapping);
} catch (IOException ex) {
throw new ElasticsearchException(ex);
}

36
x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml
View File

@ -1,8 +1,8 @@
---
"Verify data stream resolvability for xpack apis":
- skip:
version: " - 7.8.99"
reason: "data streams only supported in 7.9+"
version: " - 7.9.99"
reason: "enable in 7.9+ when backported"
features: allowed_warnings
- do:
@ -17,8 +17,7 @@
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:
@ -54,8 +53,8 @@
---
"Verify data stream resolvability in ilm explain api":
- skip:
version: " - 7.8.99"
reason: "data streams only supported in 7.9+"
version: " - 7.9.99"
reason: "enable in 7.9+ when backported"
features: allowed_warnings
- do:
@ -88,8 +87,7 @@
name: generic_logs_template
body:
index_patterns: logs-*
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
template:
settings:
index.lifecycle.name: "my_lifecycle"
@ -119,8 +117,8 @@
---
"Verify data stream resolvability rollup":
- skip:
version: " - 7.8.99"
reason: "data streams only supported in 7.9+"
version: " - 7.9.99"
reason: "enable in 7.9+ when backported"
features: allowed_warnings
- do:
@ -135,8 +133,7 @@
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:
@ -196,8 +193,7 @@
type: date
settings:
index.lifecycle.name: "my_moveable_timeseries_lifecycle"
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:
@ -261,8 +257,7 @@
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:
@ -318,8 +313,7 @@
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:
@ -358,8 +352,7 @@
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:
@ -433,8 +426,7 @@
type: keyword
price:
type: integer
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:

7
x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/10_basic.yml
View File

@ -136,8 +136,8 @@
---
"Cannot freeze write index for data stream":
- skip:
version: " - 7.8.99"
reason: "data streams only supported in 7.9+"
version: " - 7.9.99"
reason: "enable in 7.9+ when backported"
features: allowed_warnings
- do:
@ -152,8 +152,7 @@
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:

3
x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/50_data_streams.yml
View File

@ -61,8 +61,7 @@ setup:
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
---
teardown:

1
x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java
View File

@ -121,7 +121,6 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
"{\n" +
" \"index_patterns\": [ \"" + indexName + "\" ],\n" +
" \"data_stream\": {\n" +
" \"timestamp_field\": \"@timestamp\"\n" +
" },\n" +
" \"template\": \n" + Strings.toString(builder) +
"}"

7
x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml
View File

@ -68,8 +68,8 @@ setup:
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
allowed_warnings:
- "index template [my-template2] has index patterns [simple-data-stream2] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation"
@ -82,8 +82,7 @@ setup:
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'
data_stream: {}
- do:
indices.create_data_stream:

1
x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java
View File

@ -33,7 +33,6 @@ public class DataStreamsUpgradeIT extends AbstractUpgradeTestCase {
" }\n" +
" },\n" +
" \"data_stream\":{\n" +
" \"timestamp_field\":\"@timestamp\"" +
" }\n" +
" }";
Request request = new Request("PUT", "/_index_template/1");