From 5358cee29cdeed0f4b3379fc65ad35fc5e26b45a Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 14 Sep 2020 15:32:35 +0100 Subject: [PATCH] Cut over more mapping tests to MapperServiceTestCase (#62312) Shaves a few more seconds off the build. --- .../index/mapper/DynamicMappingIT.java | 9 + .../index/mapper/DynamicMappingTests.java | 778 +++++++----------- .../mapper/DynamicMappingVersionTests.java | 71 -- .../index/mapper/DynamicTemplatesTests.java | 97 +-- .../mapper/ExternalFieldMapperTests.java | 181 ++-- .../index/mapper/FieldAliasMapperTests.java | 153 ++-- .../index/mapper/MapperServiceTestCase.java | 11 + 7 files changed, 452 insertions(+), 848 deletions(-) delete mode 100644 server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingVersionTests.java diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java index c7329f228b1..e503c030bb9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java @@ -42,6 +42,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING; +import static org.hamcrest.Matchers.equalTo; public class DynamicMappingIT extends ESIntegTestCase { @@ -162,4 +163,12 @@ public class DynamicMappingIT extends ESIntegTestCase { indexingCompletedLatch.countDown(); } } + + public void testMappingVersionAfterDynamicMappingUpdate() { + createIndex("test"); + final ClusterService clusterService = internalCluster().clusterService(); + final long previousVersion = clusterService.state().metadata().index("test").getMappingVersion(); + client().prepareIndex("test", "_doc").setId("1").setSource("field", "text").get(); + assertThat(clusterService.state().metadata().index("test").getMappingVersion(), equalTo(1 + previousVersion)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java index ab7fe87ccdd..5c6f469d4aa 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java @@ -18,86 +18,55 @@ */ package org.elasticsearch.index.mapper; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import java.io.IOException; import java.time.Instant; -import java.util.Collection; -import java.util.Collections; -import static java.util.Collections.emptyMap; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; -public class DynamicMappingTests extends ESSingleNodeTestCase { +public class DynamicMappingTests extends MapperServiceTestCase { - @Override - protected Collection> getPlugins() { - return Collections.singleton(InternalSettingsPlugin.class); + private XContentBuilder dynamicMapping(String dynamicValue, CheckedConsumer buildFields) + throws IOException { + return topMapping(b -> { + b.field("dynamic", dynamicValue); + b.startObject("properties"); + buildFields.accept(b); + b.endObject(); + }); } public void testDynamicTrue() throws IOException { - String mapping = Strings.toString(jsonBuilder().startObject().startObject("type") - .field("dynamic", "true") - .startObject("properties") - .startObject("field1").field("type", "text").endObject() - .endObject() - .endObject().endObject()); + DocumentMapper defaultMapper = createDocumentMapper(dynamicMapping("true", + b -> b.startObject("field1").field("type", "text").endObject())); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); - - ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "type", "1", BytesReference - .bytes(jsonBuilder() - .startObject() - .field("field1", "value1") - .field("field2", "value2") - .endObject()), - XContentType.JSON)); + ParsedDocument doc = defaultMapper.parse(source(b -> { + b.field("field1", "value1"); + b.field("field2", "value2"); + })); assertThat(doc.rootDoc().get("field1"), equalTo("value1")); assertThat(doc.rootDoc().get("field2"), equalTo("value2")); } public void testDynamicFalse() throws IOException { - String mapping = Strings.toString(jsonBuilder().startObject().startObject("type") - .field("dynamic", "false") - .startObject("properties") - .startObject("field1").field("type", "text").endObject() - .endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createDocumentMapper(dynamicMapping("false", + b -> b.startObject("field1").field("type", "text").endObject())); - ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "type", "1", BytesReference - .bytes(jsonBuilder() - .startObject() - .field("field1", "value1") - .field("field2", "value2") - .endObject()), - XContentType.JSON)); + ParsedDocument doc = defaultMapper.parse(source(b -> { + b.field("field1", "value1"); + b.field("field2", "value2"); + })); assertThat(doc.rootDoc().get("field1"), equalTo("value1")); assertThat(doc.rootDoc().get("field2"), nullValue()); @@ -105,416 +74,263 @@ public class DynamicMappingTests extends ESSingleNodeTestCase { public void testDynamicStrict() throws IOException { - String mapping = Strings.toString(jsonBuilder().startObject().startObject("type") - .field("dynamic", "strict") - .startObject("properties") - .startObject("field1").field("type", "text").endObject() - .endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createDocumentMapper(dynamicMapping("strict", + b -> b.startObject("field1").field("type", "text").endObject())); StrictDynamicMappingException e = expectThrows(StrictDynamicMappingException.class, - () -> defaultMapper.parse(new SourceToParse("test", "type", "1", - BytesReference.bytes(jsonBuilder() - .startObject() - .field("field1", "value1") - .field("field2", "value2") - .endObject()), - XContentType.JSON))); - assertThat(e.getMessage(), equalTo("mapping set to strict, dynamic introduction of [field2] within [type] is not allowed")); + () -> defaultMapper.parse(source(b -> { + b.field("field1", "value1"); + b.field("field2", "value2"); + }))); + assertThat(e.getMessage(), equalTo("mapping set to strict, dynamic introduction of [field2] within [_doc] is not allowed")); e = expectThrows(StrictDynamicMappingException.class, - () -> defaultMapper.parse(new SourceToParse("test", "type", "1", - BytesReference.bytes(XContentFactory.jsonBuilder() - .startObject() - .field("field1", "value1") - .field("field2", (String) null) - .endObject()), - XContentType.JSON))); - assertThat(e.getMessage(), equalTo("mapping set to strict, dynamic introduction of [field2] within [type] is not allowed")); + () -> defaultMapper.parse(source(b -> { + b.field("field1", "value1"); + b.nullField("field2"); + }))); + assertThat(e.getMessage(), equalTo("mapping set to strict, dynamic introduction of [field2] within [_doc] is not allowed")); } public void testDynamicFalseWithInnerObjectButDynamicSetOnRoot() throws IOException { - String mapping = Strings.toString(jsonBuilder().startObject().startObject("type") - .field("dynamic", "false") - .startObject("properties") - .startObject("obj1").startObject("properties") - .startObject("field1").field("type", "text").endObject() - .endObject().endObject() - .endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createDocumentMapper(dynamicMapping("false", b -> { + b.startObject("obj1"); + { + b.startObject("properties"); + { + b.startObject("field1").field("type", "text").endObject(); + } + b.endObject(); + } + b.endObject(); + })); - ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "type", "1", - BytesReference.bytes(jsonBuilder() - .startObject().startObject("obj1") - .field("field1", "value1") - .field("field2", "value2") - .endObject() - .endObject()), - XContentType.JSON)); + ParsedDocument doc = defaultMapper.parse(source(b -> { + b.startObject("obj1"); + { + b.field("field1", "value1"); + b.field("field2", "value2"); + } + b.endObject(); + })); assertThat(doc.rootDoc().get("obj1.field1"), equalTo("value1")); assertThat(doc.rootDoc().get("obj1.field2"), nullValue()); } public void testDynamicStrictWithInnerObjectButDynamicSetOnRoot() throws IOException { - String mapping = Strings.toString(jsonBuilder().startObject().startObject("type") - .field("dynamic", "strict") - .startObject("properties") - .startObject("obj1").startObject("properties") - .startObject("field1").field("type", "text").endObject() - .endObject().endObject() - .endObject() - .endObject().endObject()); - DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() - .parse("type", new CompressedXContent(mapping)); + DocumentMapper defaultMapper = createDocumentMapper(dynamicMapping("strict", b -> { + b.startObject("obj1"); + { + b.startObject("properties"); + { + b.startObject("field1").field("type", "text").endObject(); + } + b.endObject(); + } + b.endObject(); + })); StrictDynamicMappingException e = expectThrows(StrictDynamicMappingException.class, () -> - defaultMapper.parse(new SourceToParse("test", "type", "1", - BytesReference.bytes(jsonBuilder() - .startObject().startObject("obj1") - .field("field1", "value1") - .field("field2", "value2") - .endObject() - .endObject()), - XContentType.JSON))); + defaultMapper.parse(source(b -> { + b.startObject("obj1"); + { + b.field("field1", "value1"); + b.field("field2", "value2"); + } + b.endObject(); + }))); assertThat(e.getMessage(), equalTo("mapping set to strict, dynamic introduction of [field2] within [obj1] is not allowed")); } public void testDynamicMappingOnEmptyString() throws Exception { - IndexService service = createIndex("test"); - client().prepareIndex("test", "type").setSource("empty_field", "").get(); - MappedFieldType fieldType = service.mapperService().fieldType("empty_field"); + MapperService mapperService = createMapperService(mapping(b -> {})); + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("empty_field", ""))); + merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate())); + MappedFieldType fieldType = mapperService.fieldType("empty_field"); assertNotNull(fieldType); } - private String serialize(ToXContent mapper) throws Exception { - XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); - mapper.toXContent(builder, new ToXContent.MapParams(emptyMap())); - return Strings.toString(builder.endObject()); - } - - private Mapper parse(DocumentMapper mapper, DocumentMapperParser parser, XContentBuilder builder) throws Exception { - IndexMetadata build = IndexMetadata.builder("") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .numberOfShards(1).numberOfReplicas(0).build(); - IndexSettings settings = new IndexSettings(build, Settings.EMPTY); - SourceToParse source = new SourceToParse("test", mapper.type(), "some_id", - BytesReference.bytes(builder), builder.contentType()); - try (XContentParser xContentParser = createParser(JsonXContent.jsonXContent, source.source())) { - ParseContext.InternalParseContext ctx = new ParseContext.InternalParseContext(settings, parser, mapper, source, xContentParser); - assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken()); - ctx.parser().nextToken(); - DocumentParser.parseObjectOrNested(ctx, mapper.root()); - Mapping mapping = DocumentParser.createDynamicUpdate(mapper.mapping(), mapper, ctx.getDynamicMappers()); - return mapping == null ? null : mapping.root(); - } - } - public void testDynamicMappingsNotNeeded() throws Exception { - IndexService indexService = createIndex("test"); - DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("foo").field("type", "text").endObject().endObject() - .endObject().endObject()); - - DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); - Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject()); - // foo is already defined in the mappings - assertNull(update); + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "text"))); + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "bar"))); + // field is already defined in mappings + assertNull(doc.dynamicMappingsUpdate()); } public void testField() throws Exception { - IndexService indexService = createIndex("test"); - DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() - .startObject("type").endObject() - .endObject()); - DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); - assertEquals(mapping, serialize(mapper)); + DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); - Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject()); - assertNotNull(update); - // original mapping not modified - assertEquals(mapping, serialize(mapper)); - // but we have an update - assertEquals(Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("foo") - .field("type", "text") - .startObject("fields") - .startObject("keyword") - .field("type", "keyword") - .field("ignore_above", 256) - .endObject() - .endObject() - .endObject() - .endObject().endObject().endObject()), serialize(update)); + ParsedDocument doc = mapper.parse(source(b -> b.field("foo", "bar"))); + assertNotNull(doc.dynamicMappingsUpdate()); + + assertEquals( + "{\"_doc\":{\"properties\":{\"foo\":{\"type\":\"text\",\"fields\":" + + "{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}}}}", + Strings.toString(doc.dynamicMappingsUpdate())); } public void testIncremental() throws Exception { - IndexService indexService = createIndex("test"); - DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); + // Make sure that mapping updates are incremental, this is important for performance otherwise // every new field introduction runs in linear time with the total number of fields - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties").startObject("foo").field("type", "text").endObject().endObject() - .endObject().endObject()); + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "text"))); - DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); - assertEquals(mapping, serialize(mapper)); + ParsedDocument doc = mapper.parse(source(b -> { + b.field("field", "bar"); + b.field("bar", "baz"); + })); + assertNotNull(doc.dynamicMappingsUpdate()); - Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar") - .field("bar", "baz").endObject()); - assertNotNull(update); - // original mapping not modified - assertEquals(mapping, serialize(mapper)); - // but we have an update - assertEquals(Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - // foo is NOT in the update - .startObject("bar").field("type", "text") - .startObject("fields") - .startObject("keyword") - .field("type", "keyword") - .field("ignore_above", 256) - .endObject() - .endObject() - .endObject() - .endObject().endObject().endObject()), serialize(update)); + assertThat(Strings.toString(doc.dynamicMappingsUpdate()), containsString("{\"bar\":")); + // field is NOT in the update + assertThat(Strings.toString(doc.dynamicMappingsUpdate()), not(containsString("{\"field\":"))); } public void testIntroduceTwoFields() throws Exception { - IndexService indexService = createIndex("test"); - DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() - .startObject("type").endObject() - .endObject()); - DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); - assertEquals(mapping, serialize(mapper)); + DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + ParsedDocument doc = mapper.parse(source(b -> { + b.field("foo", "bar"); + b.field("bar", "baz"); + })); - Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar") - .field("bar", "baz").endObject()); - assertNotNull(update); - // original mapping not modified - assertEquals(mapping, serialize(mapper)); - // but we have an update - assertEquals(Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("bar").field("type", "text") - .startObject("fields") - .startObject("keyword") - .field("type", "keyword") - .field("ignore_above", 256) - .endObject() - .endObject() - .endObject() - .startObject("foo").field("type", "text") - .startObject("fields") - .startObject("keyword") - .field("type", "keyword") - .field("ignore_above", 256) - .endObject() - .endObject() - .endObject() - .endObject().endObject().endObject()), serialize(update)); + assertNotNull(doc.dynamicMappingsUpdate()); + assertThat(Strings.toString(doc.dynamicMappingsUpdate()), containsString("\"foo\":{")); + assertThat(Strings.toString(doc.dynamicMappingsUpdate()), containsString("\"bar\":{")); } public void testObject() throws Exception { - IndexService indexService = createIndex("test"); - DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() - .startObject("type").endObject() - .endObject()); + DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + ParsedDocument doc = mapper.parse(source(b -> { + b.startObject("foo"); + { + b.startObject("bar").field("baz", "foo").endObject(); + } + b.endObject(); + })); - DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); - assertEquals(mapping, serialize(mapper)); - - Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startObject("foo").startObject("bar") - .field("baz", "foo").endObject().endObject().endObject()); - assertNotNull(update); - // original mapping not modified - assertEquals(mapping, serialize(mapper)); - // but we have an update - String serializedUpdate = serialize(update); - assertEquals(Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("foo").startObject("properties").startObject("bar").startObject("properties").startObject("baz") - .field("type", "text") - .startObject("fields").startObject("keyword").field("type", "keyword") - .field("ignore_above", 256).endObject() - .endObject().endObject().endObject().endObject().endObject().endObject() - .endObject().endObject().endObject()), serialize(update)); + assertNotNull(doc.dynamicMappingsUpdate()); + assertThat(Strings.toString(doc.dynamicMappingsUpdate()), + containsString("{\"foo\":{\"properties\":{\"bar\":{\"properties\":{\"baz\":{\"type\":\"text\"")); } public void testArray() throws Exception { - IndexService indexService = createIndex("test"); - DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() - .startObject("type").endObject() - .endObject()); - DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); - assertEquals(mapping, serialize(mapper)); + DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + ParsedDocument doc = mapper.parse(source(b -> b.startArray("foo").value("bar").value("baz").endArray())); - Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject() - .startArray("foo").value("bar").value("baz").endArray().endObject()); - assertNotNull(update); - // original mapping not modified - assertEquals(mapping, serialize(mapper)); - // but we have an update - assertEquals(Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("foo") - .field("type", "text") - .startObject("fields") - .startObject("keyword") - .field("type", "keyword") - .field("ignore_above", 256) - .endObject() - .endObject() - .endObject() - .endObject().endObject().endObject()), serialize(update)); + assertNotNull(doc.dynamicMappingsUpdate()); + assertThat(Strings.toString(doc.dynamicMappingsUpdate()), + containsString("{\"foo\":{\"type\":\"text\"")); } public void testInnerDynamicMapping() throws Exception { - IndexService indexService = createIndex("test"); - DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") - .startObject("foo").field("type", "object").endObject() - .endObject().endObject().endObject()); - DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); - assertEquals(mapping, serialize(mapper)); + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "object"))); + ParsedDocument doc = mapper.parse(source(b -> { + b.startObject("field"); + { + b.startObject("bar").field("baz", "foo").endObject(); + } + b.endObject(); + })); - Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startObject("foo") - .startObject("bar").field("baz", "foo").endObject().endObject().endObject()); - assertNotNull(update); - // original mapping not modified - assertEquals(mapping, serialize(mapper)); - // but we have an update - assertEquals(Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("foo").startObject("properties").startObject("bar").startObject("properties") - .startObject("baz").field("type", "text").startObject("fields") - .startObject("keyword").field("type", "keyword").field("ignore_above", 256).endObject() - .endObject().endObject().endObject().endObject().endObject().endObject() - .endObject().endObject().endObject()), serialize(update)); + assertNotNull(doc.dynamicMappingsUpdate()); + assertThat(Strings.toString(doc.dynamicMappingsUpdate()), + containsString("{\"field\":{\"properties\":{\"bar\":{\"properties\":{\"baz\":{\"type\":\"text\"")); } public void testComplexArray() throws Exception { - IndexService indexService = createIndex("test"); - DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() - .startObject("type").endObject() - .endObject()); - DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); - assertEquals(mapping, serialize(mapper)); - - Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startArray("foo") - .startObject().field("bar", "baz").endObject() - .startObject().field("baz", 3).endObject() - .endArray().endObject()); - assertEquals(mapping, serialize(mapper)); - assertEquals(Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("foo").startObject("properties") - .startObject("bar").field("type", "text") - .startObject("fields") - .startObject("keyword") - .field("type", "keyword") - .field("ignore_above", 256) - .endObject() - .endObject() - .endObject() - .startObject("baz").field("type", "long").endObject() - .endObject().endObject() - .endObject().endObject().endObject()), serialize(update)); + DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + ParsedDocument doc = mapper.parse(source(b -> { + b.startArray("foo"); + { + b.startObject().field("bar", "baz").endObject(); + b.startObject().field("baz", 3).endObject(); + } + b.endArray(); + })); + assertNotNull(doc.dynamicMappingsUpdate()); + assertEquals("{\"_doc\":{\"properties\":{\"foo\":{\"properties\":{\"bar\":{\"type\":\"text\",\"fields\":{" + + "\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"baz\":{\"type\":\"long\"}}}}}}", + Strings.toString(doc.dynamicMappingsUpdate())); } public void testReuseExistingMappings() throws Exception { - IndexService indexService = createIndex("test", Settings.EMPTY, "type", - "my_field1", "type=text,store=true", - "my_field2", "type=integer,store=false", - "my_field3", "type=long,doc_values=false", - "my_field4", "type=float,index=false", - "my_field5", "type=double,store=true", - "my_field6", "type=date,doc_values=false", - "my_field7", "type=boolean,doc_values=false"); - // Even if the dynamic type of our new field is long, we already have a mapping for the same field // of type string so it should be mapped as a string - DocumentMapper newMapper = indexService.mapperService().documentMapperWithAutoCreate("type").getDocumentMapper(); - Mapper update = parse(newMapper, indexService.mapperService().documentMapperParser(), - XContentFactory.jsonBuilder().startObject() - .field("my_field1", 42) - .field("my_field2", 43) - .field("my_field3", 44) - .field("my_field4", 45) - .field("my_field5", 46) - .field("my_field6", Instant.now().toEpochMilli()) - .field("my_field7", true) - .endObject()); - assertNull(update); + DocumentMapper newMapper = createDocumentMapper(mapping(b -> { + b.startObject("my_field1").field("type", "text").field("store", "true").endObject(); + b.startObject("my_field2").field("type", "integer").field("store", "false").endObject(); + b.startObject("my_field3").field("type", "long").field("doc_values", "false").endObject(); + b.startObject("my_field4").field("type", "float").field("index", "false").endObject(); + b.startObject("my_field5").field("type", "double").field("store", "true").endObject(); + b.startObject("my_field6").field("type", "date").field("doc_values", "false").endObject(); + b.startObject("my_field7").field("type", "boolean").field("doc_values", "false").endObject(); + })); - MapperParsingException e = expectThrows(MapperParsingException.class, () -> { - parse(newMapper, indexService.mapperService().documentMapperParser(), - XContentFactory.jsonBuilder().startObject().field("my_field2", "foobar").endObject()); - }); + ParsedDocument doc = newMapper.parse(source(b -> { + b.field("my_field1", 42); + b.field("my_field2", 43); + b.field("my_field3", 44); + b.field("my_field4", 45); + b.field("my_field5", 46); + b.field("my_field6", Instant.now().toEpochMilli()); + b.field("my_field7", true); + })); + assertNull(doc.dynamicMappingsUpdate()); + + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> newMapper.parse(source(b -> b.field("my_field2", "foobar")))); assertThat(e.getMessage(), containsString("failed to parse field [my_field2] of type [integer]")); } public void testMixTemplateMultiFieldAndMappingReuse() throws Exception { - IndexService indexService = createIndex("test"); - XContentBuilder mappings1 = jsonBuilder().startObject() - .startObject("_doc") - .startArray("dynamic_templates") - .startObject() - .startObject("template1") - .field("match_mapping_type", "string") - .startObject("mapping") - .field("type", "text") - .startObject("fields") - .startObject("raw") - .field("type", "keyword") - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - .endArray() - .endObject().endObject(); - indexService.mapperService().merge("_doc", new CompressedXContent(BytesReference.bytes(mappings1)), - MapperService.MergeReason.MAPPING_UPDATE); + MapperService mapperService = createMapperService(topMapping(b -> { + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("template1"); + { + b.field("match_mapping_type", "string"); + b.startObject("mapping"); + { + b.field("type", "text"); + b.startObject("fields"); + { + b.startObject("raw").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + assertNull(mapperService.documentMapper().mappers().getMapper("field.raw")); - XContentBuilder json = XContentFactory.jsonBuilder().startObject() - .field("field", "foo") - .endObject(); - SourceToParse source = new SourceToParse("test", "_doc", "1", BytesReference.bytes(json), json.contentType()); - DocumentMapper mapper = indexService.mapperService().documentMapper("_doc"); - assertNull(mapper.mappers().getMapper("field.raw")); - ParsedDocument parsed = mapper.parse(source); + ParsedDocument parsed = mapperService.documentMapper().parse(source(b -> b.field("field", "foo"))); assertNotNull(parsed.dynamicMappingsUpdate()); - indexService.mapperService().merge("_doc", new CompressedXContent(parsed.dynamicMappingsUpdate().toString()), - MapperService.MergeReason.MAPPING_UPDATE); - mapper = indexService.mapperService().documentMapper("_doc"); - assertNotNull(mapper.mappers().getMapper("field.raw")); - parsed = mapper.parse(source); + merge(mapperService, dynamicMapping(parsed.dynamicMappingsUpdate())); + assertNotNull(mapperService.documentMapper().mappers().getMapper("field.raw")); + parsed = mapperService.documentMapper().parse(source(b -> b.field("field", "foo"))); assertNull(parsed.dynamicMappingsUpdate()); } public void testDefaultFloatingPointMappings() throws IOException { - MapperService mapperService = createIndex("test").mapperService(); - String mapping = Strings.toString(jsonBuilder().startObject() - .startObject("type") - .field("numeric_detection", true) - .endObject().endObject()); - mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); - DocumentMapper mapper = mapperService.documentMapper("type"); + DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("numeric_detection", true))); doTestDefaultFloatingPointMappings(mapper, XContentFactory.jsonBuilder()); doTestDefaultFloatingPointMappings(mapper, XContentFactory.yamlBuilder()); doTestDefaultFloatingPointMappings(mapper, XContentFactory.smileBuilder()); @@ -528,7 +344,7 @@ public class DynamicMappingTests extends ESSingleNodeTestCase { .field("baz", (double) 3.2f) // double that can be accurately represented as a float .field("quux", "3.2") // float detected through numeric detection .endObject()); - ParsedDocument parsedDocument = mapper.parse(new SourceToParse("index", "type", "id", + ParsedDocument parsedDocument = mapper.parse(new SourceToParse("index", "_doc", "id", source, builder.contentType())); Mapping update = parsedDocument.dynamicMappingsUpdate(); assertNotNull(update); @@ -539,107 +355,84 @@ public class DynamicMappingTests extends ESSingleNodeTestCase { } public void testNumericDetectionEnabled() throws Exception { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .field("numeric_detection", true) - .endObject().endObject()); + MapperService mapperService = createMapperService(topMapping(b -> b.field("numeric_detection", true))); - IndexService index = createIndex("test"); - client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping, XContentType.JSON).get(); - DocumentMapper defaultMapper = index.mapperService().documentMapper("type"); - - ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "type", "1", BytesReference - .bytes(XContentFactory.jsonBuilder() - .startObject() - .field("s_long", "100") - .field("s_double", "100.0") - .endObject()), - XContentType.JSON)); + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> { + b.field("s_long", "100"); + b.field("s_double", "100.0"); + })); assertNotNull(doc.dynamicMappingsUpdate()); - client().admin().indices().preparePutMapping("test").setType("type") - .setSource(doc.dynamicMappingsUpdate().toString(), XContentType.JSON).get(); + merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate())); - defaultMapper = index.mapperService().documentMapper("type"); - Mapper mapper = defaultMapper.mappers().getMapper("s_long"); + Mapper mapper = mapperService.documentMapper().mappers().getMapper("s_long"); assertThat(mapper.typeName(), equalTo("long")); - mapper = defaultMapper.mappers().getMapper("s_double"); + mapper = mapperService.documentMapper().mappers().getMapper("s_double"); assertThat(mapper.typeName(), equalTo("float")); } public void testNumericDetectionDefault() throws Exception { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .endObject().endObject()); - IndexService index = createIndex("test"); - client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping, XContentType.JSON).get(); - DocumentMapper defaultMapper = index.mapperService().documentMapper("type"); + MapperService mapperService = createMapperService(mapping(b -> {})); - ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "type", "1", BytesReference - .bytes(XContentFactory.jsonBuilder() - .startObject() - .field("s_long", "100") - .field("s_double", "100.0") - .endObject()), - XContentType.JSON)); + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> { + b.field("s_long", "100"); + b.field("s_double", "100.0"); + })); assertNotNull(doc.dynamicMappingsUpdate()); - assertAcked(client().admin().indices().preparePutMapping("test").setType("type") - .setSource(doc.dynamicMappingsUpdate().toString(), XContentType.JSON).get()); + merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate())); - defaultMapper = index.mapperService().documentMapper("type"); - Mapper mapper = defaultMapper.mappers().getMapper("s_long"); + Mapper mapper = mapperService.documentMapper().mappers().getMapper("s_long"); assertThat(mapper, instanceOf(TextFieldMapper.class)); - mapper = defaultMapper.mappers().getMapper("s_double"); + mapper = mapperService.documentMapper().mappers().getMapper("s_double"); assertThat(mapper, instanceOf(TextFieldMapper.class)); } public void testDateDetectionInheritsFormat() throws Exception { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startArray("dynamic_date_formats") - .value("yyyy-MM-dd") - .endArray() - .startArray("dynamic_templates") - .startObject() - .startObject("dates") - .field("match_mapping_type", "date") - .field("match", "*2") - .startObject("mapping") - .endObject() - .endObject() - .endObject() - .startObject() - .startObject("dates") - .field("match_mapping_type", "date") - .field("match", "*3") - .startObject("mapping") - .field("format", "yyyy-MM-dd||epoch_millis") - .endObject() - .endObject() - .endObject() - .endArray() - .endObject().endObject()); - IndexService index = createIndex("test"); - client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping, XContentType.JSON).get(); - DocumentMapper defaultMapper = index.mapperService().documentMapper("type"); + MapperService mapperService = createMapperService(topMapping(b -> { + b.startArray("dynamic_date_formats").value("yyyy-MM-dd").endArray(); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("dates"); + { + b.field("match_mapping_type", "date"); + b.field("match", "*2"); + b.startObject("mapping").endObject(); + } + b.endObject(); + } + b.endObject(); + b.startObject(); + { + b.startObject("dates"); + { + b.field("match_mapping_type", "date"); + b.field("match", "*3"); + b.startObject("mapping").field("format", "yyyy-MM-dd||epoch_millis").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); - ParsedDocument doc = defaultMapper.parse(new SourceToParse("test", "type", "1", BytesReference - .bytes(XContentFactory.jsonBuilder() - .startObject() - .field("date1", "2016-11-20") - .field("date2", "2016-11-20") - .field("date3", "2016-11-20") - .endObject()), - XContentType.JSON)); + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> { + b.field("date1", "2016-11-20"); + b.field("date2", "2016-11-20"); + b.field("date3", "2016-11-20"); + })); assertNotNull(doc.dynamicMappingsUpdate()); - assertAcked(client().admin().indices().preparePutMapping("test").setType("type") - .setSource(doc.dynamicMappingsUpdate().toString(), XContentType.JSON).get()); - defaultMapper = index.mapperService().documentMapper("type"); + merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate())); - DateFieldMapper dateMapper1 = (DateFieldMapper) defaultMapper.mappers().getMapper("date1"); - DateFieldMapper dateMapper2 = (DateFieldMapper) defaultMapper.mappers().getMapper("date2"); - DateFieldMapper dateMapper3 = (DateFieldMapper) defaultMapper.mappers().getMapper("date3"); + DateFieldMapper dateMapper1 = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("date1"); + DateFieldMapper dateMapper2 = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("date2"); + DateFieldMapper dateMapper3 = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("date3"); // inherited from dynamic date format assertEquals("yyyy-MM-dd", dateMapper1.fieldType().dateTimeFormatter().pattern()); // inherited from dynamic date format since the mapping in the template did not specify a format @@ -651,37 +444,36 @@ public class DynamicMappingTests extends ESSingleNodeTestCase { public void testDynamicTemplateOrder() throws IOException { // https://github.com/elastic/elasticsearch/issues/18625 // elasticsearch used to apply templates that do not have a match_mapping_type first - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startArray("dynamic_templates") - .startObject() - .startObject("type-based") - .field("match_mapping_type", "string") - .startObject("mapping") - .field("type", "keyword") - .endObject() - .endObject() - .endObject() - .startObject() - .startObject("path-based") - .field("path_match", "foo") - .startObject("mapping") - .field("type", "long") - .endObject() - .endObject() - .endObject() - .endArray() - .endObject().endObject(); - IndexService index = createIndex("test", Settings.EMPTY, "type", mapping); - client().prepareIndex("test", "type", "1").setSource("foo", "abc").get(); - assertThat(index.mapperService().fieldType("foo"), instanceOf(KeywordFieldMapper.KeywordFieldType.class)); - } - - public void testMappingVersionAfterDynamicMappingUpdate() { - createIndex("test", client().admin().indices().prepareCreate("test").addMapping("type")); - final ClusterService clusterService = getInstanceFromNode(ClusterService.class); - final long previousVersion = clusterService.state().metadata().index("test").getMappingVersion(); - client().prepareIndex("test", "type", "1").setSource("field", "text").get(); - assertThat(clusterService.state().metadata().index("test").getMappingVersion(), equalTo(1 + previousVersion)); + MapperService mapperService = createMapperService(topMapping(b -> { + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("type-based"); + { + b.field("match_mapping_type", "string"); + b.startObject("mapping").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + b.startObject(); + { + b.startObject("path-based"); + { + b.field("path_match", "foo"); + b.startObject("mapping").field("type", "long").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("foo", "abc"))); + assertNotNull(doc.dynamicMappingsUpdate()); + merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate())); + assertThat(mapperService.fieldType("foo"), instanceOf(KeywordFieldMapper.KeywordFieldType.class)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingVersionTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingVersionTests.java deleted file mode 100644 index 6b451b64db0..00000000000 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingVersionTests.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper; - -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; - -import java.io.IOException; -import java.util.Collection; - -public class DynamicMappingVersionTests extends ESSingleNodeTestCase { - - @Override - protected Collection> getPlugins() { - return pluginList(InternalSettingsPlugin.class); - } - - public void testDynamicMappingDefault() throws IOException { - MapperService mapperService = createIndex("my-index").mapperService(); - DocumentMapper documentMapper = mapperService - .documentMapperWithAutoCreate("my-type").getDocumentMapper(); - - ParsedDocument parsedDoc = documentMapper.parse( - new SourceToParse("my-index", "my-type", "1", BytesReference - .bytes(XContentFactory.jsonBuilder() - .startObject() - .field("foo", 3) - .endObject()), XContentType.JSON)); - - String expectedMapping = Strings.toString(XContentFactory.jsonBuilder().startObject() - .startObject("my-type") - .startObject("properties") - .startObject("foo").field("type", "long") - .endObject().endObject().endObject().endObject()); - assertEquals(expectedMapping, parsedDoc.dynamicMappingsUpdate().toString()); - } - - public void testDynamicMappingSettingRemoval() { - Settings settings = Settings.builder() - .put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), false) - .build(); - Exception e = expectThrows(IllegalArgumentException.class, () -> createIndex("test-index", settings)); - assertEquals(e.getMessage(), "Setting index.mapper.dynamic was removed after version 6.0.0"); - assertSettingDeprecationsAndWarnings(new Setting[] { MapperService.INDEX_MAPPER_DYNAMIC_SETTING }); - } - -} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java index 48f7924df51..4ba6aa01cbd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java @@ -22,38 +22,37 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.ParseContext.Document; -import org.elasticsearch.test.ESSingleNodeTestCase; -import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; -public class DynamicTemplatesTests extends ESSingleNodeTestCase { - public void testMatchTypeOnly() throws Exception { - XContentBuilder builder = JsonXContent.contentBuilder(); - builder.startObject().startObject("person").startArray("dynamic_templates").startObject().startObject("test") - .field("match_mapping_type", "string") - .startObject("mapping").field("index", false).endObject() - .endObject().endObject().endArray().endObject().endObject(); - IndexService index = createIndex("test"); - client().admin().indices().preparePutMapping("test").setType("person").setSource(builder).get(); +public class DynamicTemplatesTests extends MapperServiceTestCase { - MapperService mapperService = index.mapperService(); - DocumentMapper docMapper = mapperService.documentMapper("person"); - builder = JsonXContent.contentBuilder(); - builder.startObject().field("s", "hello").field("l", 1).endObject(); - ParsedDocument parsedDoc = docMapper.parse(new SourceToParse("test", "person", "1", BytesReference.bytes(builder), - XContentType.JSON)); - client().admin().indices().preparePutMapping("test").setType("person") - .setSource(parsedDoc.dynamicMappingsUpdate().toString(), XContentType.JSON).get(); + public void testMatchTypeOnly() throws Exception { + MapperService mapperService = createMapperService(topMapping(b -> { + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match_mapping_type", "string"); + b.startObject("mapping").field("index", false).endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + DocumentMapper docMapper = mapperService.documentMapper(); + ParsedDocument parsedDoc = docMapper.parse(source(b -> { + b.field("s", "hello"); + b.field("l", 1); + })); + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); assertThat(mapperService.fieldType("s"), notNullValue()); assertFalse(mapperService.fieldType("s").isSearchable()); @@ -64,15 +63,11 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase { public void testSimple() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/simple/test-mapping.json"); - IndexService index = createIndex("test"); - client().admin().indices().preparePutMapping("test").setType("person").setSource(mapping, XContentType.JSON).get(); - DocumentMapper docMapper = index.mapperService().documentMapper("person"); - byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/simple/test-data.json"); - ParsedDocument parsedDoc = docMapper.parse(new SourceToParse("test", "person", "1", new BytesArray(json), - XContentType.JSON)); - client().admin().indices().preparePutMapping("test").setType("person") - .setSource(parsedDoc.dynamicMappingsUpdate().toString(), XContentType.JSON).get(); - docMapper = index.mapperService().documentMapper("person"); + MapperService mapperService = createMapperService("person", mapping); + String docJson = copyToStringFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/simple/test-data.json"); + ParsedDocument parsedDoc = mapperService.documentMapper().parse(source(docJson)); + + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); Document doc = parsedDoc.rootDoc(); IndexableField f = doc.getField("name"); @@ -81,7 +76,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase { assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions()); assertThat(f.fieldType().tokenized(), equalTo(false)); - Mapper fieldMapper = docMapper.mappers().getMapper("name"); + Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper("name"); assertNotNull(fieldMapper); f = doc.getField("multi1"); @@ -90,7 +85,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase { assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions()); assertThat(f.fieldType().tokenized(), equalTo(true)); - fieldMapper = docMapper.mappers().getMapper("multi1"); + fieldMapper = mapperService.documentMapper().mappers().getMapper("multi1"); assertNotNull(fieldMapper); f = doc.getField("multi1.org"); @@ -99,7 +94,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase { assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions()); assertThat(f.fieldType().tokenized(), equalTo(false)); - fieldMapper = docMapper.mappers().getMapper("multi1.org"); + fieldMapper = mapperService.documentMapper().mappers().getMapper("multi1.org"); assertNotNull(fieldMapper); f = doc.getField("multi2"); @@ -108,7 +103,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase { assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions()); assertThat(f.fieldType().tokenized(), equalTo(true)); - fieldMapper = docMapper.mappers().getMapper("multi2"); + fieldMapper = mapperService.documentMapper().mappers().getMapper("multi2"); assertNotNull(fieldMapper); f = doc.getField("multi2.org"); @@ -117,21 +112,17 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase { assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions()); assertThat(f.fieldType().tokenized(), equalTo(false)); - fieldMapper = docMapper.mappers().getMapper("multi2.org"); + fieldMapper = mapperService.documentMapper().mappers().getMapper("multi2.org"); assertNotNull(fieldMapper); } public void testSimpleWithXContentTraverse() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/simple/test-mapping.json"); - IndexService index = createIndex("test"); - client().admin().indices().preparePutMapping("test").setType("person").setSource(mapping, XContentType.JSON).get(); - DocumentMapper docMapper = index.mapperService().documentMapper("person"); - byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/simple/test-data.json"); - ParsedDocument parsedDoc = docMapper.parse(new SourceToParse("test", "person", "1", new BytesArray(json), - XContentType.JSON)); - client().admin().indices().preparePutMapping("test").setType("person") - .setSource(parsedDoc.dynamicMappingsUpdate().toString(), XContentType.JSON).get(); - docMapper = index.mapperService().documentMapper("person"); + MapperService mapperService = createMapperService("person", mapping); + String docJson = copyToStringFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/simple/test-data.json"); + ParsedDocument parsedDoc = mapperService.documentMapper().parse(source(docJson)); + + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); Document doc = parsedDoc.rootDoc(); IndexableField f = doc.getField("name"); @@ -140,7 +131,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase { assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions()); assertThat(f.fieldType().tokenized(), equalTo(false)); - Mapper fieldMapper = docMapper.mappers().getMapper("name"); + Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper("name"); assertNotNull(fieldMapper); f = doc.getField("multi1"); @@ -149,7 +140,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase { assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions()); assertThat(f.fieldType().tokenized(), equalTo(true)); - fieldMapper = docMapper.mappers().getMapper("multi1"); + fieldMapper = mapperService.documentMapper().mappers().getMapper("multi1"); assertNotNull(fieldMapper); f = doc.getField("multi1.org"); @@ -158,7 +149,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase { assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions()); assertThat(f.fieldType().tokenized(), equalTo(false)); - fieldMapper = docMapper.mappers().getMapper("multi1.org"); + fieldMapper = mapperService.documentMapper().mappers().getMapper("multi1.org"); assertNotNull(fieldMapper); f = doc.getField("multi2"); @@ -167,7 +158,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase { assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions()); assertThat(f.fieldType().tokenized(), equalTo(true)); - fieldMapper = docMapper.mappers().getMapper("multi2"); + fieldMapper = mapperService.documentMapper().mappers().getMapper("multi2"); assertNotNull(fieldMapper); f = doc.getField("multi2.org"); @@ -176,7 +167,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase { assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions()); assertThat(f.fieldType().tokenized(), equalTo(false)); - fieldMapper = docMapper.mappers().getMapper("multi2.org"); + fieldMapper = mapperService.documentMapper().mappers().getMapper("multi2.org"); assertNotNull(fieldMapper); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java index fc37785cb25..0d5f908ece6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java @@ -21,77 +21,28 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.indices.mapper.MapperRegistry; -import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; -import org.elasticsearch.test.VersionUtils; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Supplier; -import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -public class ExternalFieldMapperTests extends ESSingleNodeTestCase { +public class ExternalFieldMapperTests extends MapperServiceTestCase { @Override - protected Collection> getPlugins() { - return pluginList(InternalSettingsPlugin.class); - } - - @Override - protected boolean forbidPrivateIndexSettings() { - return false; + protected Collection getPlugins() { + return Collections.singletonList(new ExternalMapperPlugin()); } public void testExternalValues() throws Exception { - Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, - Version.CURRENT); - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build(); - IndexService indexService = createIndex("test", settings); - MapperRegistry mapperRegistry = new MapperRegistry( - singletonMap(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")), - singletonMap(ExternalMetadataMapper.CONTENT_TYPE, ExternalMetadataMapper.PARSER), MapperPlugin.NOOP_FIELD_FILTER); - Supplier queryShardContext = () -> { - return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null); - }; - DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), - indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext, null); - DocumentMapper documentMapper = parser.parse("type", new CompressedXContent( - Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject(ExternalMetadataMapper.CONTENT_TYPE) - .endObject() - .startObject("properties") - .startObject("field").field("type", "external").endObject() - .endObject() - .endObject().endObject()) - )); + DocumentMapper documentMapper = createDocumentMapper(fieldMapping(b -> b.field("type", "external"))); - ParsedDocument doc = documentMapper.parse(new SourceToParse("test", "type", "1", BytesReference - .bytes(XContentFactory.jsonBuilder() - .startObject() - .field("field", "1234") - .endObject()), - XContentType.JSON)); + ParsedDocument doc = documentMapper.parse(source(b -> b.field("field", "1234"))); assertThat(doc.rootDoc().getField("field.bool"), notNullValue()); assertThat(doc.rootDoc().getField("field.bool").stringValue(), is("T")); @@ -111,45 +62,32 @@ public class ExternalFieldMapperTests extends ESSingleNodeTestCase { } public void testExternalValuesWithMultifield() throws Exception { - IndexService indexService = createIndex("test"); - Map mapperParsers = new HashMap<>(); - mapperParsers.put(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")); - mapperParsers.put(TextFieldMapper.CONTENT_TYPE, new TextFieldMapper.TypeParser()); - mapperParsers.put(KeywordFieldMapper.CONTENT_TYPE, KeywordFieldMapper.PARSER); - MapperRegistry mapperRegistry = new MapperRegistry(mapperParsers, Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER); - Supplier queryShardContext = () -> { - return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null); - }; - DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), - indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext, null); + DocumentMapper documentMapper = createDocumentMapper(fieldMapping(b -> { + b.field("type", ExternalMapperPlugin.EXTERNAL); + b.startObject("fields"); + { + b.startObject("text"); + { + b.field("type", "text"); + b.field("store", true); + b.startObject("fields"); + { + b.startObject("raw"); + { + b.field("type", "keyword"); + b.field("store", true); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + })); - DocumentMapper documentMapper = parser.parse("type", new CompressedXContent( - Strings - .toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("field") - .field("type", ExternalMapperPlugin.EXTERNAL) - .startObject("fields") - .startObject("text") - .field("type", "text") - .field("store", true) - .startObject("fields") - .startObject("raw") - .field("type", "keyword") - .field("store", true) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - .endObject().endObject().endObject()))); - - ParsedDocument doc = documentMapper.parse(new SourceToParse("test", "type", "1", BytesReference - .bytes(XContentFactory.jsonBuilder() - .startObject() - .field("field", "1234") - .endObject()), - XContentType.JSON)); + ParsedDocument doc = documentMapper.parse(source(b -> b.field("field", "1234"))); assertThat(doc.rootDoc().getField("field.bool"), notNullValue()); assertThat(doc.rootDoc().getField("field.bool").stringValue(), is("T")); @@ -179,49 +117,28 @@ public class ExternalFieldMapperTests extends ESSingleNodeTestCase { } public void testExternalValuesWithMultifieldTwoLevels() throws Exception { - IndexService indexService = createIndex("test"); - Map mapperParsers = new HashMap<>(); - mapperParsers.put(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")); - mapperParsers.put(ExternalMapperPlugin.EXTERNAL_BIS, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "bar")); - mapperParsers.put(TextFieldMapper.CONTENT_TYPE, new TextFieldMapper.TypeParser()); - MapperRegistry mapperRegistry = new MapperRegistry(mapperParsers, Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER); - Supplier queryShardContext = () -> { - return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null); - }; - DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), - indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext, null); + DocumentMapper documentMapper = createDocumentMapper(fieldMapping(b -> { + b.field("type", ExternalMapperPlugin.EXTERNAL); + b.startObject("fields"); + { + b.startObject("text"); + { + b.field("type", "text"); + b.startObject("fields"); + { + b.startObject("generated").field("type", ExternalMapperPlugin.EXTERNAL_BIS).endObject(); + b.startObject("raw").field("type", "text").endObject(); + } + b.endObject(); + } + b.endObject(); + b.startObject("raw").field("type", "text").endObject(); + } + b.endObject(); + })); - DocumentMapper documentMapper = parser.parse("type", new CompressedXContent( - Strings - .toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") - .startObject("field") - .field("type", ExternalMapperPlugin.EXTERNAL) - .startObject("fields") - .startObject("text") - .field("type", "text") - .startObject("fields") - .startObject("generated") - .field("type", ExternalMapperPlugin.EXTERNAL_BIS) - .endObject() - .startObject("raw") - .field("type", "text") - .endObject() - .endObject() - .endObject() - .startObject("raw") - .field("type", "text") - .endObject() - .endObject() - .endObject() - .endObject().endObject().endObject()))); - - ParsedDocument doc = documentMapper.parse(new SourceToParse("test", "type", "1", BytesReference - .bytes(XContentFactory.jsonBuilder() - .startObject() - .field("field", "1234") - .endObject()), - XContentType.JSON)); + ParsedDocument doc = documentMapper.parse(source(b -> b.field("field", "1234"))); assertThat(doc.rootDoc().getField("field.bool"), notNullValue()); assertThat(doc.rootDoc().getField("field.bool").stringValue(), is("T")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperTests.java index 6107eff3ac2..182ee52d367 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperTests.java @@ -20,30 +20,16 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.mapper.MapperService.MergeReason; -import org.elasticsearch.test.ESSingleNodeTestCase; -import org.junit.Before; import java.io.IOException; -public class FieldAliasMapperTests extends ESSingleNodeTestCase { - private MapperService mapperService; - private DocumentMapperParser parser; - - @Before - public void setup() { - IndexService indexService = createIndex("test"); - mapperService = indexService.mapperService(); - parser = mapperService.documentMapperParser(); - } +public class FieldAliasMapperTests extends MapperServiceTestCase { public void testParsing() throws IOException { String mapping = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject("_doc") .startObject("properties") .startObject("alias-field") .field("type", "alias") @@ -55,79 +41,56 @@ public class FieldAliasMapperTests extends ESSingleNodeTestCase { .endObject() .endObject() .endObject()); - DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + DocumentMapper mapper = createDocumentMapper("_doc", mapping); assertEquals(mapping, mapper.mappingSource().toString()); } - public void testParsingWithMissingPath() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder() - .startObject() - .startObject("type") - .startObject("properties") - .startObject("alias-field") - .field("type", "alias") - .endObject() - .endObject() - .endObject() - .endObject()); + public void testParsingWithMissingPath() { MapperParsingException exception = expectThrows(MapperParsingException.class, - () -> parser.parse("type", new CompressedXContent(mapping))); - assertEquals("The [path] property must be specified for field [alias-field].", exception.getMessage()); + () -> createDocumentMapper(mapping(b -> b.startObject("alias-field").field("type", "alias").endObject()))); + assertEquals("Failed to parse mapping [_doc]: The [path] property must be specified for field [alias-field].", + exception.getMessage()); } - public void testParsingWithExtraArgument() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder() - .startObject() - .startObject("type") - .startObject("properties") - .startObject("alias-field") - .field("type", "alias") - .field("path", "concrete-field") - .field("extra-field", "extra-value") - .endObject() - .endObject() - .endObject() - .endObject()); - MapperParsingException exception = expectThrows(MapperParsingException.class, - () -> parser.parse("type", new CompressedXContent(mapping))); - assertEquals("Mapping definition for [alias-field] has unsupported parameters: [extra-field : extra-value]", + public void testParsingWithExtraArgument() { + MapperParsingException exception = expectThrows(MapperParsingException.class, () -> createDocumentMapper(mapping(b -> { + b.startObject("alias-field"); + { + b.field("type", "alias"); + b.field("path", "concrete-field"); + b.field("extra-field", "extra-value"); + } + b.endObject(); + }))); + assertEquals("Failed to parse mapping [_doc]: " + + "Mapping definition for [alias-field] has unsupported parameters: [extra-field : extra-value]", exception.getMessage()); } public void testMerge() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() - .startObject("type") - .startObject("properties") - .startObject("first-field") - .field("type", "keyword") - .endObject() - .startObject("alias-field") - .field("type", "alias") - .field("path", "first-field") - .endObject() - .endObject() - .endObject() - .endObject()); - mapperService.merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("first-field").field("type", "keyword").endObject(); + b.startObject("alias-field"); + { + b.field("type", "alias"); + b.field("path", "first-field"); + } + b.endObject(); + })); MappedFieldType firstFieldType = mapperService.fieldType("alias-field"); assertEquals("first-field", firstFieldType.name()); assertTrue(firstFieldType instanceof KeywordFieldMapper.KeywordFieldType); - String newMapping = Strings.toString(XContentFactory.jsonBuilder().startObject() - .startObject("type") - .startObject("properties") - .startObject("second-field") - .field("type", "text") - .endObject() - .startObject("alias-field") - .field("type", "alias") - .field("path", "second-field") - .endObject() - .endObject() - .endObject() - .endObject()); - mapperService.merge("type", new CompressedXContent(newMapping), MergeReason.MAPPING_UPDATE); + merge(mapperService, mapping(b -> { + b.startObject("second-field").field("type", "text").endObject(); + b.startObject("alias-field"); + { + b.field("type", "alias"); + b.field("path", "second-field"); + } + b.endObject(); + })); MappedFieldType secondFieldType = mapperService.fieldType("alias-field"); assertEquals("second-field", secondFieldType.name()); @@ -135,32 +98,24 @@ public class FieldAliasMapperTests extends ESSingleNodeTestCase { } public void testMergeFailure() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() - .startObject("type") - .startObject("properties") - .startObject("concrete-field") - .field("type", "text") - .endObject() - .startObject("alias-field") - .field("type", "alias") - .field("path", "concrete-field") - .endObject() - .endObject() - .endObject() - .endObject()); - mapperService.merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); - String newMapping = Strings.toString(XContentFactory.jsonBuilder().startObject() - .startObject("type") - .startObject("properties") - .startObject("alias-field") - .field("type", "keyword") - .endObject() - .endObject() - .endObject() - .endObject()); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, - () -> mapperService.merge("type", new CompressedXContent(newMapping), MergeReason.MAPPING_UPDATE)); + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("concrete-field").field("type", "text").endObject(); + b.startObject("alias-field"); + { + b.field("type", "alias"); + b.field("path", "concrete-field"); + } + b.endObject(); + })); + + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> merge(mapperService, mapping(b -> { + b.startObject("alias-field"); + { + b.field("type", "keyword"); + } + b.endObject(); + }))); assertEquals("Cannot merge a field alias mapping [alias-field] with a mapping that is not for a field alias.", exception.getMessage()); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index cf405cbb1d5..c933555da69 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -27,6 +27,7 @@ import org.apache.lucene.store.Directory; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; @@ -104,6 +105,12 @@ public abstract class MapperServiceTestCase extends ESTestCase { return createMapperService(getIndexSettings(), mappings); } + protected final MapperService createMapperService(String type, String mappings) throws IOException { + MapperService mapperService = createMapperService(mapping(b -> {})); + merge(type, mapperService, mappings); + return mapperService; + } + /** * Create a {@link MapperService} like we would for an index. */ @@ -160,6 +167,10 @@ public abstract class MapperServiceTestCase extends ESTestCase { return new SourceToParse("test", "_doc", "1", BytesReference.bytes(builder), XContentType.JSON); } + protected final SourceToParse source(String source) { + return new SourceToParse("test", "_doc", "1", new BytesArray(source), XContentType.JSON); + } + /** * Merge a new mapping into the one in the provided {@link MapperService}. */