From 252ae5f15aa35149fade936bbed285d13d2597fe Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 22 Mar 2016 19:38:26 +0100 Subject: [PATCH] Upgrade dynamic templates that use a dynamic type. #17254 Now that string has been splitted into text and keyword, we use text as a dynamic type when encountering string fields in a json document. However this does not play well with existing templates that look like ``` { "mapping": { "index": "not_analyzed", "type": "{dynamic_type}" }, "match": "*" } ``` Since we want existing templates to keep working as much as possible in 5.0, this commit adds a hack to dynamic templates so that elasticsearch will create a keyword field if the `index` property is set and is either `no` or `not_analyzed`, similarly to what was done in #16991. While this will make upgrades easier, we still need to figure out a way to allow users to create keyword fields when using dynamic types. --- .../index/mapper/object/DynamicTemplate.java | 26 ++++- .../core/StringMappingUpgradeTests.java | 95 +++++++++++++++++++ 2 files changed, 119 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java b/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java index 44cdac17be1..4a1ab2027b7 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java @@ -161,8 +161,30 @@ public class DynamicTemplate implements ToXContent { } public String mappingType(String dynamicType) { - return mapping.containsKey("type") ? mapping.get("type").toString().replace("{dynamic_type}", dynamicType).replace("{dynamicType}", dynamicType) : dynamicType; - } + String type; + if (mapping.containsKey("type")) { + type = mapping.get("type").toString(); + type = type.replace("{dynamic_type}", dynamicType); + type = type.replace("{dynamicType}", dynamicType); + } else { + type = dynamicType; + } + if (type.equals(mapping.get("type")) == false // either the type was not set, or we updated it through replacements + && "text".equals(type)) { // and the result is "text" + // now that string has been splitted into text and keyword, we use text for + // dynamic mappings. However before it used to be possible to index as a keyword + // by setting index=not_analyzed, so for now we will use a keyword field rather + // than a text field if index=not_analyzed and the field type was not specified + // explicitly + // TODO: remove this in 6.0 + // TODO: how to do it in the future? + final Object index = mapping.get("index"); + if ("not_analyzed".equals(index) || "no".equals(index)) { + type = "keyword"; + } + } + return type; + } private boolean patternMatch(String pattern, String str) { if (matchType == MatchType.SIMPLE) { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java index 4d7effbf41f..09fd07265cc 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java @@ -24,6 +24,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.index.IndexOptions; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -33,6 +34,8 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.core.TextFieldMapper.TextFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; @@ -239,4 +242,96 @@ public class StringMappingUpgradeTests extends ESSingleNodeTestCase { } } } + + public void testUpgradeTemplateWithDynamicType() throws IOException { + IndexService indexService = createIndex("test"); + DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startArray("dynamic_templates") + .startObject() + .startObject("my_template") + .field("match_mapping_type", "string") + .startObject("mapping") + .field("store", true) + .endObject() + .endObject() + .endObject() + .endArray() + .endObject().endObject().string(); + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + BytesReference source = XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject().bytes(); + ParsedDocument doc = mapper.parse("test", "type", "id", source); + Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo"); + assertThat(fooMapper, instanceOf(TextFieldMapper.class)); + assertTrue(((TextFieldMapper) fooMapper).fieldType().stored()); + } + + public void testUpgradeTemplateWithDynamicType2() throws IOException { + IndexService indexService = createIndex("test"); + DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startArray("dynamic_templates") + .startObject() + .startObject("my_template") + .field("match_mapping_type", "string") + .startObject("mapping") + .field("type", "{dynamic_type}") + .field("store", true) + .endObject() + .endObject() + .endObject() + .endArray() + .endObject().endObject().string(); + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + BytesReference source = XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject().bytes(); + ParsedDocument doc = mapper.parse("test", "type", "id", source); + Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo"); + assertThat(fooMapper, instanceOf(TextFieldMapper.class)); + assertTrue(((TextFieldMapper) fooMapper).fieldType().stored()); + } + + public void testUpgradeTemplateWithDynamicTypeKeyword() throws IOException { + IndexService indexService = createIndex("test"); + DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startArray("dynamic_templates") + .startObject() + .startObject("my_template") + .field("match_mapping_type", "string") + .startObject("mapping") + .field("index", "not_analyzed") + .endObject() + .endObject() + .endObject() + .endArray() + .endObject().endObject().string(); + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + BytesReference source = XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject().bytes(); + ParsedDocument doc = mapper.parse("test", "type", "id", source); + Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo"); + assertThat(fooMapper, instanceOf(KeywordFieldMapper.class)); + } + + public void testUpgradeTemplateWithDynamicTypeKeyword2() throws IOException { + IndexService indexService = createIndex("test"); + DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startArray("dynamic_templates") + .startObject() + .startObject("my_template") + .field("match_mapping_type", "string") + .startObject("mapping") + .field("type", "{dynamic_type}") + .field("index", "not_analyzed") + .endObject() + .endObject() + .endObject() + .endArray() + .endObject().endObject().string(); + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + BytesReference source = XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject().bytes(); + ParsedDocument doc = mapper.parse("test", "type", "id", source); + Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo"); + assertThat(fooMapper, instanceOf(KeywordFieldMapper.class)); + } }