honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enforce valid field mapping exists for timestamp_field in templates. (#58036) 

Backport of #57741 to 7.x branch.

Relates to #53100
This commit is contained in:
Martijn van Groningen 2020-06-12 15:24:42 +02:00 committed by GitHub
parent a5a251d8c0
commit 01d8bb8cfa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 310 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
docs/reference/indices/create-data-stream.asciidoc
View File

@ -31,6 +31,15 @@ template exists with a `data_stream` definition.
PUT _index_template/template
{
"index_patterns": ["my-data-stream*"],
"template": {
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
}
}
}
},
"data_stream": {
"timestamp_field": "@timestamp"
}

9
docs/reference/indices/delete-data-stream.asciidoc
View File

@ -12,6 +12,15 @@ Deletes an existing data stream along with its backing indices.
PUT _index_template/template
{
"index_patterns": ["my-data-stream*"],
"template": {
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
}
}
}
},
"data_stream": {
"timestamp_field": "@timestamp"
}

9
docs/reference/indices/get-data-stream.asciidoc
View File

@ -12,6 +12,15 @@ Returns information about one or more data streams.
PUT _index_template/template
{
"index_patterns": ["my-data-stream*"],
"template": {
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
}
}
}
},
"data_stream": {
"timestamp_field": "@timestamp"
}

9
docs/reference/indices/rollover-index.asciidoc
View File

@ -232,6 +232,15 @@ The API returns the following response:
PUT _index_template/template
{
"index_patterns": ["my-data-stream*"],
"template": {
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
}
}
}
},
"data_stream": {
"timestamp_field": "@timestamp"
}

15
rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
View File

@ -8,6 +8,11 @@ setup:
name: my-template1
body:
index_patterns: [simple-data-stream1]
template:
mappings:
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'
- do:
@ -17,6 +22,11 @@ setup:
name: my-template2
body:
index_patterns: [simple-data-stream2]
template:
mappings:
properties:
'@timestamp2':
type: date
data_stream:
timestamp_field: '@timestamp2'
@ -215,6 +225,11 @@ setup:
name: generic_logs_template
body:
index_patterns: logs-*
template:
mappings:
properties:
'timestamp':
type: date
data_stream:
timestamp_field: timestamp

5
rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml
View File

@ -12,6 +12,11 @@
name: my-template
body:
index_patterns: [logs-*]
template:
mappings:
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'

5
rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/40_supported_apis.yml
View File

@ -9,6 +9,11 @@ setup:
name: logs_template
body:
index_patterns: logs-foobar
template:
mappings:
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'

5
rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml
View File

@ -8,6 +8,11 @@ setup:
name: my-template
body:
index_patterns: [simple-*]
template:
mappings:
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'

5
rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml
View File

@ -12,6 +12,11 @@
name: my-template
body:
index_patterns: [data-*]
template:
mappings:
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'

5
rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/10_basic.yml
View File

@ -133,6 +133,11 @@
name: my-template1
body:
index_patterns: [simple-data-stream1]
template:
mappings:
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'

5
rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml
View File

@ -12,6 +12,11 @@
name: my-template
body:
index_patterns: [data-*]
template:
mappings:
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'

5
server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java
View File

@ -40,8 +40,10 @@ import org.elasticsearch.action.support.replication.ReplicationRequest;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
import org.elasticsearch.cluster.metadata.Template;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
@ -65,6 +67,7 @@ import static org.elasticsearch.action.DocWriteRequest.OpType.CREATE;
import static org.elasticsearch.action.DocWriteResponse.Result.CREATED;
import static org.elasticsearch.action.DocWriteResponse.Result.UPDATED;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamServiceTests.generateMapping;
import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.arrayWithSize;
@ -224,7 +227,7 @@ public class BulkIntegrationIT extends ESIntegTestCase {
createTemplateRequest.indexTemplate(
new ComposableIndexTemplate(
Collections.singletonList("logs-foo*"),
null,
new Template(null, new CompressedXContent(generateMapping("@timestamp")), null),
null, null, null, null,
new ComposableIndexTemplate.DataStreamTemplate("@timestamp"))
);

47
server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java
View File

@ -47,6 +47,7 @@ import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamServiceTests;
import org.elasticsearch.cluster.metadata.Template;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.xcontent.ObjectPath;
@ -244,6 +245,9 @@ public class DataStreamIT extends ESIntegTestCase {
" \"properties\": {\n" +
" \"baz_field\": {\n" +
" \"type\": \"keyword\"\n" +
" },\n" +
" \"@timestamp\": {\n" +
" \"type\": \"date\"\n" +
" }\n" +
" }\n" +
" }";
@ -306,6 +310,46 @@ public class DataStreamIT extends ESIntegTestCase {
DataStream.getDefaultBackingIndexName(dataStreamName, 2))).actionGet());
}
public void testTimeStampValidationNoFieldMapping() throws Exception {
// Adding a template without a mapping for timestamp field and expect template creation to fail.
PutComposableIndexTemplateAction.Request createTemplateRequest = new PutComposableIndexTemplateAction.Request("logs-foo");
createTemplateRequest.indexTemplate(
new ComposableIndexTemplate(
Collections.singletonList("logs-*"),
new Template(null, new CompressedXContent("{}"), null),
null, null, null, null,
new ComposableIndexTemplate.DataStreamTemplate("@timestamp"))
);
Exception e = expectThrows(IllegalArgumentException.class,
() -> client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet());
assertThat(e.getCause().getCause().getMessage(), equalTo("expected timestamp field [@timestamp], but found no timestamp field"));
}
public void testTimeStampValidationInvalidFieldMapping() throws Exception {
// Adding a template with an invalid mapping for timestamp field and expect template creation to fail.
String mapping = "{\n" +
" \"properties\": {\n" +
" \"@timestamp\": {\n" +
" \"type\": \"keyword\"\n" +
" }\n" +
" }\n" +
" }";
PutComposableIndexTemplateAction.Request createTemplateRequest = new PutComposableIndexTemplateAction.Request("logs-foo");
createTemplateRequest.indexTemplate(
new ComposableIndexTemplate(
Collections.singletonList("logs-*"),
new Template(null, new CompressedXContent(mapping), null),
null, null, null, null,
new ComposableIndexTemplate.DataStreamTemplate("@timestamp"))
);
Exception e = expectThrows(IllegalArgumentException.class,
() -> client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet());
assertThat(e.getCause().getCause().getMessage(), equalTo("expected timestamp field [@timestamp] to be of types " +
"[date, date_nanos], but instead found type [keyword]"));
}
public void testDataStreamsResolvability() throws Exception {
createIndexTemplate("id", "logs-*", "ts");
String dataStreamName = "logs-foobar";
@ -469,7 +513,8 @@ public class DataStreamIT extends ESIntegTestCase {
request.indexTemplate(
new ComposableIndexTemplate(
Collections.singletonList(pattern),
null,
new Template(null,
new CompressedXContent(MetadataCreateDataStreamServiceTests.generateMapping(timestampFieldName)), null),
null, null, null, null,
new ComposableIndexTemplate.DataStreamTemplate(timestampFieldName))
);

20
server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java
View File

@ -31,17 +31,25 @@ import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest;
import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.collect.List;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.mapper.DateFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.threadpool.ThreadPool;
import java.util.LinkedHashSet;
import java.util.Collections;
import java.util.Locale;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
public class MetadataCreateDataStreamService {
private static final Logger logger = LogManager.getLogger(MetadataCreateDataStreamService.class);
private static final Set<String> ALLOWED_TIMESTAMPFIELD_TYPES =
new LinkedHashSet<>(List.of(DateFieldMapper.CONTENT_TYPE, DateFieldMapper.DATE_NANOS_CONTENT_TYPE));
private final ClusterService clusterService;
private final ActiveShardsObserver activeShardsObserver;
@ -159,4 +167,16 @@ public class MetadataCreateDataStreamService {
return composableIndexTemplate;
}
public static void validateTimestampFieldMapping(String timestampFieldName, MapperService mapperService) {
MappedFieldType timestampFieldMapper = mapperService.fieldType(timestampFieldName);
if (timestampFieldMapper == null) {
throw new IllegalArgumentException("expected timestamp field [" + timestampFieldName + "], but found no timestamp field");
}
String type = timestampFieldMapper.typeName();
if (ALLOWED_TIMESTAMPFIELD_TYPES.contains(type) == false) {
throw new IllegalArgumentException("expected timestamp field [" + timestampFieldName + "] to be of types " +
ALLOWED_TIMESTAMPFIELD_TYPES + ", but instead found type [" + type + "]");
}
}
}

5
server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
View File

@ -72,6 +72,7 @@ import java.util.TreeSet;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping;
import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED;
/**
@ -1028,6 +1029,10 @@ public class MetadataIndexTemplateService {
MapperService dummyMapperService = tempIndexService.mapperService();
// TODO: Eventually change this to use MergeReason.INDEX_TEMPLATE
dummyMapperService.merge(finalMappings, MergeReason.MAPPING_UPDATE);
if (template.getDataStreamTemplate() != null) {
String tsFieldName = template.getDataStreamTemplate().getTimestampField();
validateTimestampFieldMapping(tsFieldName, dummyMapperService);
}
} catch (Exception e) {
throw new IllegalArgumentException("invalid composite mappings for [" + templateName + "]", e);
}

19
server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
View File

@ -47,13 +47,16 @@ import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.RoutingFieldMapper;
import org.elasticsearch.index.shard.IndexEventListener;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.InvalidIndexNameException;
@ -71,6 +74,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamServiceTests.generateMapping;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@ -543,7 +547,14 @@ public class MetadataRolloverServiceTests extends ESTestCase {
when(env.sharedDataFile()).thenReturn(null);
AllocationService allocationService = mock(AllocationService.class);
when(allocationService.reroute(any(ClusterState.class), any(String.class))).then(i -> i.getArguments()[0]);
IndicesService indicesService = mockIndicesServices();
DocumentMapper documentMapper = mock(DocumentMapper.class);
when(documentMapper.type()).thenReturn("_doc");
CompressedXContent mapping = new CompressedXContent(generateMapping(dataStream.getTimeStampField()));
when(documentMapper.mappingSource()).thenReturn(mapping);
RoutingFieldMapper routingFieldMapper = mock(RoutingFieldMapper.class);
when(routingFieldMapper.required()).thenReturn(false);
when(documentMapper.routingFieldMapper()).thenReturn(routingFieldMapper);
IndicesService indicesService = mockIndicesServices(documentMapper);
IndexNameExpressionResolver mockIndexNameExpressionResolver = mock(IndexNameExpressionResolver.class);
when(mockIndexNameExpressionResolver.resolveDateMathExpression(any())).then(returnsFirstArg());
@ -688,6 +699,10 @@ public class MetadataRolloverServiceTests extends ESTestCase {
}
private IndicesService mockIndicesServices() throws Exception {
return mockIndicesServices(null);
}
private IndicesService mockIndicesServices(DocumentMapper documentMapper) throws Exception {
/*
* Throws Exception because Eclipse uses the lower bound for
* CheckedFunction's exception type so it thinks the "when" call
@ -702,7 +717,7 @@ public class MetadataRolloverServiceTests extends ESTestCase {
when(indexService.index()).thenReturn(indexMetadata.getIndex());
MapperService mapperService = mock(MapperService.class);
when(indexService.mapperService()).thenReturn(mapperService);
when(mapperService.documentMapper()).thenReturn(null);
when(mapperService.documentMapper()).thenReturn(documentMapper);
when(indexService.getIndexEventListener()).thenReturn(new IndexEventListener() {});
when(indexService.getIndexSortSupplier()).thenReturn(() -> null);
//noinspection unchecked

21
server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java
View File

@ -69,12 +69,14 @@ public class ComposableIndexTemplateTests extends AbstractDiffableSerializationT
CompressedXContent mappings = null;
Map<String, AliasMetadata> aliases = null;
Template template = null;
if (randomBoolean()) {
ComposableIndexTemplate.DataStreamTemplate dataStreamTemplate = randomDataStreamTemplate();
if (dataStreamTemplate != null || randomBoolean()) {
if (randomBoolean()) {
settings = randomSettings();
}
if (randomBoolean()) {
mappings = randomMappings();
if (dataStreamTemplate != null || randomBoolean()) {
mappings = randomMappings(dataStreamTemplate);
}
if (randomBoolean()) {
aliases = randomAliases();
@ -87,8 +89,6 @@ public class ComposableIndexTemplateTests extends AbstractDiffableSerializationT
meta = randomMeta();
}
ComposableIndexTemplate.DataStreamTemplate dataStreamTemplate = randomDataStreamTemplate();
List<String> indexPatterns = randomList(1, 4, () -> randomAlphaOfLength(4));
List<String> componentTemplates = randomList(0, 10, () -> randomAlphaOfLength(5));
return new ComposableIndexTemplate(indexPatterns,
@ -111,9 +111,13 @@ public class ComposableIndexTemplateTests extends AbstractDiffableSerializationT
return Collections.singletonMap(aliasName, aliasMeta);
}
private static CompressedXContent randomMappings() {
private static CompressedXContent randomMappings(ComposableIndexTemplate.DataStreamTemplate dataStreamTemplate) {
try {
return new CompressedXContent("{\"properties\":{\"" + randomAlphaOfLength(5) + "\":{\"type\":\"keyword\"}}}");
if (dataStreamTemplate != null) {
return new CompressedXContent("{\"properties\":{\"" + dataStreamTemplate.getTimestampField() + "\":{\"type\":\"date\"}}}");
} else {
return new CompressedXContent("{\"properties\":{\"" + randomAlphaOfLength(5) + "\":{\"type\":\"keyword\"}}}");
}
} catch (IOException e) {
fail("got an IO exception creating fake mappings: " + e);
return null;
@ -162,7 +166,8 @@ public class ComposableIndexTemplateTests extends AbstractDiffableSerializationT
orig.priority(), orig.version(), orig.metadata(), orig.getDataStreamTemplate());
case 1:
return new ComposableIndexTemplate(orig.indexPatterns(),
randomValueOtherThan(orig.template(), () -> new Template(randomSettings(), randomMappings(), randomAliases())),
randomValueOtherThan(orig.template(), () -> new Template(randomSettings(),
randomMappings(orig.getDataStreamTemplate()), randomAliases())),
orig.composedOf(),
orig.priority(),
orig.version(),

86
server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java
View File

@ -25,11 +25,15 @@ import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.MapperTestUtils;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.util.Collections;
import static org.elasticsearch.cluster.DataStreamTestHelper.createFirstBackingIndex;
import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
@ -146,6 +150,59 @@ public class MetadataCreateDataStreamServiceTests extends ESTestCase {
return MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req);
}
public void testValidateTimestampFieldMapping() throws Exception {
String mapping = generateMapping("@timestamp", "date");
validateTimestampFieldMapping("@timestamp", createMapperService(mapping));
mapping = generateMapping("@timestamp", "date_nanos");
validateTimestampFieldMapping("@timestamp", createMapperService(mapping));
}
public void testValidateTimestampFieldMappingNoFieldMapping() {
Exception e = expectThrows(IllegalArgumentException.class,
() -> validateTimestampFieldMapping("@timestamp", createMapperService("{}")));
assertThat(e.getMessage(),
equalTo("expected timestamp field [@timestamp], but found no timestamp field"));
String mapping = generateMapping("@timestamp2", "date");
e = expectThrows(IllegalArgumentException.class,
() -> validateTimestampFieldMapping("@timestamp", createMapperService(mapping)));
assertThat(e.getMessage(),
equalTo("expected timestamp field [@timestamp], but found no timestamp field"));
}
public void testValidateTimestampFieldMappingInvalidFieldType() {
String mapping = generateMapping("@timestamp", "keyword");
Exception e = expectThrows(IllegalArgumentException.class,
() -> validateTimestampFieldMapping("@timestamp", createMapperService(mapping)));
assertThat(e.getMessage(), equalTo("expected timestamp field [@timestamp] to be of types [date, date_nanos], " +
"but instead found type [keyword]"));
}
public void testValidateNestedTimestampFieldMapping() throws Exception {
String fieldType = randomBoolean() ? "date" : "date_nanos";
String mapping = "{\n" +
" \"properties\": {\n" +
" \"event\": {\n" +
" \"properties\": {\n" +
" \"@timestamp\": {\n" +
" \"type\": \"" + fieldType + "\"\n" +
" },\n" +
" \"another_field\": {\n" +
" \"type\": \"keyword\"\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
" }";
MapperService mapperService = createMapperService(mapping);
validateTimestampFieldMapping("event.@timestamp", mapperService);
Exception e = expectThrows(IllegalArgumentException.class,
() -> validateTimestampFieldMapping("event.another_field", mapperService));
assertThat(e.getMessage(), equalTo("expected timestamp field [event.another_field] to be of types [date, date_nanos], " +
"but instead found type [keyword]"));
}
private static MetadataCreateIndexService getMetadataCreateIndexService() throws Exception {
MetadataCreateIndexService s = mock(MetadataCreateIndexService.class);
when(s.applyCreateIndexRequest(any(ClusterState.class), any(CreateIndexClusterStateUpdateRequest.class), anyBoolean()))
@ -159,6 +216,7 @@ public class MetadataCreateDataStreamServiceTests extends ESTestCase {
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
.put(request.settings())
.build())
.putMapping("_doc", generateMapping("@timestamp"))
.numberOfShards(1)
.numberOfReplicas(1)
.build(), false);
@ -168,4 +226,32 @@ public class MetadataCreateDataStreamServiceTests extends ESTestCase {
return s;
}
public static String generateMapping(String timestampFieldName) {
return generateMapping(timestampFieldName, "date");
}
static String generateMapping(String timestampFieldName, String type) {
return "{\n" +
" \"properties\": {\n" +
" \"" + timestampFieldName + "\": {\n" +
" \"type\": \"" + type + "\"\n" +
" }\n" +
" }\n" +
" }";
}
MapperService createMapperService(String mapping) throws IOException {
String indexName = "test";
IndexMetadata indexMetadata = IndexMetadata.builder(indexName)
.settings(Settings.builder()
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1))
.putMapping("_doc", mapping)
.build();
MapperService mapperService =
MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY, indexName);
mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_UPDATE);
return mapperService;
}
}

37
x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java
View File

@ -9,6 +9,7 @@ package org.elasticsearch.xpack.ilm;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.Template;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.test.rest.ESRestTestCase;
@ -42,8 +43,15 @@ public class TimeSeriesDataStreamsIT extends ESRestTestCase {
String policyName = "logs-policy";
createNewSingletonPolicy(client(), policyName, "hot", new RolloverAction(null, null, 1L));
String mapping = "{\n" +
" \"properties\": {\n" +
" \"@timestamp\": {\n" +
" \"type\": \"date\"\n" +
" }\n" +
" }\n" +
" }";
Settings lifecycleNameSetting = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyName).build();
Template template = new Template(lifecycleNameSetting, null, null);
Template template = new Template(lifecycleNameSetting, new CompressedXContent(mapping), null);
createComposableTemplate(client(), "logs-template", "logs-foo*", template);
String dataStream = "logs-foo";
@ -60,11 +68,18 @@ public class TimeSeriesDataStreamsIT extends ESRestTestCase {
String policyName = "logs-policy";
createNewSingletonPolicy(client(), policyName, "warm", new ShrinkAction(1));
String mapping = "{\n" +
" \"properties\": {\n" +
" \"@timestamp\": {\n" +
" \"type\": \"date\"\n" +
" }\n" +
" }\n" +
" }";
Settings settings = Settings.builder()
.put(LifecycleSettings.LIFECYCLE_NAME, policyName)
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3)
.build();
Template template = new Template(settings, null, null);
Template template = new Template(settings, new CompressedXContent(mapping), null);
createComposableTemplate(client(), "logs-template", "logs-foo*", template);
String dataStream = "logs-foo";
@ -89,11 +104,18 @@ public class TimeSeriesDataStreamsIT extends ESRestTestCase {
String policyName = "logs-policy";
createFullPolicy(client(), policyName, TimeValue.ZERO);
String mapping = "{\n" +
" \"properties\": {\n" +
" \"@timestamp\": {\n" +
" \"type\": \"date\"\n" +
" }\n" +
" }\n" +
" }";
Settings settings = Settings.builder()
.put(LifecycleSettings.LIFECYCLE_NAME, policyName)
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3)
.build();
Template template = new Template(settings, null, null);
Template template = new Template(settings, new CompressedXContent(mapping), null);
createComposableTemplate(client(), "logs-template", "logs-foo*", template);
String dataStream = "logs-foo";
@ -115,11 +137,18 @@ public class TimeSeriesDataStreamsIT extends ESRestTestCase {
String policyName = "logs-policy";
createNewSingletonPolicy(client(), policyName, "cold", new SearchableSnapshotAction(snapshotRepo));
String mapping = "{\n" +
" \"properties\": {\n" +
" \"@timestamp\": {\n" +
" \"type\": \"date\"\n" +
" }\n" +
" }\n" +
" }";
Settings settings = Settings.builder()
.put(LifecycleSettings.LIFECYCLE_NAME, policyName)
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3)
.build();
Template template = new Template(settings, null, null);
Template template = new Template(settings, new CompressedXContent(mapping), null);
createComposableTemplate(client(), "logs-template", "logs-foo*", template);
String dataStream = "logs-foo";
indexDocument(client(), dataStream, true);

5
x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml
View File

@ -12,6 +12,11 @@
name: my-template
body:
index_patterns: [logs-*]
template:
mappings:
properties:
'@timestamp':
type: date
data_stream:
timestamp_field: '@timestamp'