Add validation for dynamic templates (#52890)

Backport of #51233 to the seven dot x branch.

Tries to load a `Mapper` instance for the mapping snippet of a dynamic template.
This should catch things like using an analyzer that is undefined or mapping attributes that are unused.

This is best effort:
* If `{{name}}` placeholder is used in the mapping snippet then validation is skipped.
* If `match_mapping_type` is not specified then validation is performed for all mapping types.
  If parsing succeeds with a single mapping type then this the dynamic mapping is considered valid.

If is detected that a dynamic template mapping snippet is invalid at mapping update time then the mapping update is failed for indices created on 8.0.0-alpha1 and later. For indices created on prior version a deprecation warning is omitted instead. In 7.x clusters the mapping update will never fail in case of an invalid dynamic template mapping snippet and a deprecation warning will always be omitted.

Closes #17411
Closes #24419

Co-authored-by: Adrien Grand <jpountz@gmail.com>

docs/reference/mapping/dynamic/templates.asciidoc

@@ -37,6 +37,19 @@ Dynamic templates are specified as an array of named objects:
 <2> The match conditions can include any of : `match_mapping_type`, `match`, `match_pattern`, `unmatch`, `path_match`, `path_unmatch`.
 <3> The mapping that the matched field should use.
 
+If a provided mapping contains an invalid mapping snippet then that results in
+a validation error. Validation always occurs when applying the dynamic template
+at index time or in most cases when updating the dynamic template.
+
+Whether updating the dynamic template fails when supplying an invalid mapping snippet depends on the following:
+* If no `match_mapping_type` has been specified then if the template is valid with one predefined mapping type then
+  the mapping snippet is considered valid. However if at index time a field that matches with the template is indexed
+  as a different type then an validation error will occur at index time instead. For example configuring a dynamic
+  template with no `match_mapping_type` is considered valid as string type, but at index time a field that matches with
+  the dynamic template is indexed as a long, then at index time a validation error may still occur.
+* If the `{{name}}` placeholder is used in the mapping snippet then the validation is skipped when updating
+  the dynamic template. This is because the field name is unknown at that time. The validation will then occur
+  when applying the template at index time.
 
 Templates are processed in order -- the first matching template wins. When
 putting new dynamic templates through the <<indices-put-mapping, put mapping>> API,
@@ -409,4 +422,3 @@ PUT my_index
 
 <1> Like the default dynamic mapping rules, doubles are mapped as floats, which
     are usually accurate enough, yet require half the disk space.
-

docs/reference/release-notes/7.7.asciidoc

@@ -0,0 +1,13 @@
+[[release-notes-7.7.0]]
+== {es} version 7.7.0
+
+coming[7.7.0]
+
+[[breaking-7.7.0]]
+[float]
+=== Breaking changes
+
+Mapping::
+* Dynamic mappings in indices created on 8.0 and later have stricter validation at mapping update time and
+  results in a deprecation warning for indices created in Elasticsearch 7.7.0 and later.
+  (e.g. incorrect analyzer settings or unknown field types). {pull}51233[#51233]

server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java

@@ -354,6 +354,18 @@ public class DynamicTemplate implements ToXContentObject {
         return processedList;
     }
 
+    String getName() {
+        return name;
+    }
+
+    XContentFieldType getXContentFieldType() {
+        return xcontentFieldType;
+    }
+
+    Map<String, Object> getMapping() {
+        return mapping;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -886,4 +886,5 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         }
         return reloadedAnalyzers;
     }
+
 }

server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java

@@ -19,9 +19,13 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -34,6 +38,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
@@ -41,6 +46,9 @@ import static org.elasticsearch.index.mapper.TypeParsers.parseDateTimeFormatter;
 
 public class RootObjectMapper extends ObjectMapper {
 
+    private static final Logger LOGGER = LogManager.getLogger(RootObjectMapper.class);
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LOGGER);
+
     public static class Defaults {
         public static final DateFormatter[] DYNAMIC_DATE_TIME_FORMATTERS =
                 new DateFormatter[]{
@@ -128,7 +136,7 @@ public class RootObjectMapper extends ObjectMapper {
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder)
-                        || processField(builder, fieldName, fieldNode, parserContext.indexVersionCreated())) {
+                        || processField(builder, fieldName, fieldNode, parserContext)) {
                     iterator.remove();
                 }
             }
@@ -136,7 +144,7 @@ public class RootObjectMapper extends ObjectMapper {
         }
 
         protected boolean processField(RootObjectMapper.Builder builder, String fieldName, Object fieldNode,
-                Version indexVersionCreated) {
+                ParserContext parserContext) {
             if (fieldName.equals("date_formats") || fieldName.equals("dynamic_date_formats")) {
                 if (fieldNode instanceof List) {
                     List<DateFormatter> formatters = new ArrayList<>();
@@ -159,7 +167,7 @@ public class RootObjectMapper extends ObjectMapper {
                 //          "template_1" : {
                 //              "match" : "*_test",
                 //              "match_mapping_type" : "string",
-                //              "mapping" : { "type" : "string", "store" : "yes" }
+                //              "mapping" : { "type" : "keyword", "store" : "yes" }
                 //          }
                 //      }
                 //  ]
@@ -176,8 +184,9 @@ public class RootObjectMapper extends ObjectMapper {
                     Map.Entry<String, Object> entry = tmpl.entrySet().iterator().next();
                     String templateName = entry.getKey();
                     Map<String, Object> templateParams = (Map<String, Object>) entry.getValue();
-                    DynamicTemplate template = DynamicTemplate.parse(templateName, templateParams, indexVersionCreated);
+                    DynamicTemplate template = DynamicTemplate.parse(templateName, templateParams, parserContext.indexVersionCreated());
                     if (template != null) {
+                        validateDynamicTemplate(parserContext, template);
                         templates.add(template);
                     }
                 }
@@ -326,4 +335,111 @@ public class RootObjectMapper extends ObjectMapper {
             builder.field("numeric_detection", numericDetection.value());
         }
     }
+
+    private static void validateDynamicTemplate(Mapper.TypeParser.ParserContext parserContext,
+                                                DynamicTemplate dynamicTemplate) {
+
+        if (containsSnippet(dynamicTemplate.getMapping(), "{name}")) {
+            // Can't validate template, because field names can't be guessed up front.
+            return;
+        }
+
+        final XContentFieldType[] types;
+        if (dynamicTemplate.getXContentFieldType() != null) {
+            types = new XContentFieldType[]{dynamicTemplate.getXContentFieldType()};
+        } else {
+            types = XContentFieldType.values();
+        }
+
+        Exception lastError = null;
+        boolean dynamicTemplateInvalid = true;
+
+        for (XContentFieldType contentFieldType : types) {
+            String defaultDynamicType = contentFieldType.defaultMappingType();
+            String mappingType = dynamicTemplate.mappingType(defaultDynamicType);
+            Mapper.TypeParser typeParser = parserContext.typeParser(mappingType);
+            if (typeParser == null) {
+                lastError = new IllegalArgumentException("No mapper found for type [" + mappingType + "]");
+                continue;
+            }
+
+            Map<String, Object> fieldTypeConfig = dynamicTemplate.mappingForName("__dummy__", defaultDynamicType);
+            fieldTypeConfig.remove("type");
+            try {
+                Mapper.Builder<?, ?> dummyBuilder = typeParser.parse("__dummy__", fieldTypeConfig, parserContext);
+                if (fieldTypeConfig.isEmpty()) {
+                    Settings indexSettings = parserContext.mapperService().getIndexSettings().getSettings();
+                    BuilderContext builderContext = new BuilderContext(indexSettings, new ContentPath(1));
+                    dummyBuilder.build(builderContext);
+                    dynamicTemplateInvalid = false;
+                    break;
+                } else {
+                    lastError = new IllegalArgumentException("Unused mapping attributes [" + fieldTypeConfig + "]");
+                }
+            } catch (Exception e) {
+                lastError = e;
+            }
+        }
+
+        final boolean shouldEmitDeprecationWarning = parserContext.indexVersionCreated().onOrAfter(Version.V_7_7_0);
+        if (dynamicTemplateInvalid && shouldEmitDeprecationWarning) {
+            String message = String.format(Locale.ROOT, "dynamic template [%s] has invalid content [%s]",
+                dynamicTemplate.getName(), Strings.toString(dynamicTemplate));
+
+            final String deprecationMessage;
+            if (lastError != null) {
+                deprecationMessage = String.format(Locale.ROOT, "%s, caused by [%s]", message, lastError.getMessage());
+            } else {
+                deprecationMessage = message;
+            }
+            DEPRECATION_LOGGER.deprecatedAndMaybeLog("invalid_dynamic_template", deprecationMessage);
+        }
+    }
+
+    private static boolean containsSnippet(Map<?, ?> map, String snippet) {
+        for (Map.Entry<?, ?> entry : map.entrySet()) {
+            String key = entry.getKey().toString();
+            if (key.contains(snippet)) {
+                return true;
+            }
+
+            Object value = entry.getValue();
+            if (value instanceof Map) {
+                if (containsSnippet((Map<?, ?>) value, snippet)) {
+                    return true;
+                }
+            } else if (value instanceof List) {
+                if (containsSnippet((List<?>) value, snippet)) {
+                    return true;
+                }
+            } else if (value instanceof String) {
+                String valueString = (String) value;
+                if (valueString.contains(snippet)) {
+                    return true;
+                }
+            }
+        }
+
+        return false;
+    }
+
+    private static boolean containsSnippet(List<?> list, String snippet) {
+        for (Object value : list) {
+            if (value instanceof Map) {
+                if (containsSnippet((Map<?, ?>) value, snippet)) {
+                    return true;
+                }
+            } else if (value instanceof List) {
+                if (containsSnippet((List<?>) value, snippet)) {
+                    return true;
+                }
+            } else if (value instanceof String) {
+                String valueString = (String) value;
+                if (valueString.contains(snippet)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
 }

server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java

@@ -19,14 +19,21 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import java.util.Arrays;
 
+import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
+import static org.hamcrest.Matchers.containsString;
+
 public class RootObjectMapperTests extends ESSingleNodeTestCase {
 
     public void testNumericDetection() throws Exception {
@@ -200,4 +207,193 @@ public class RootObjectMapperTests extends ESSingleNodeTestCase {
                     () -> parser.parse("type", new CompressedXContent(mapping)));
         assertEquals("Dynamic template syntax error. An array of named objects is expected.", e.getMessage());
     }
+
+    public void testIllegalDynamicTemplateUnknownFieldType() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder();
+        mapping.startObject();
+        {
+            mapping.startObject("type");
+            mapping.startArray("dynamic_templates");
+            {
+                mapping.startObject();
+                mapping.startObject("my_template");
+                mapping.field("match_mapping_type", "string");
+                mapping.startObject("mapping");
+                mapping.field("type", "string");
+                mapping.endObject();
+                mapping.endObject();
+                mapping.endObject();
+            }
+            mapping.endArray();
+            mapping.endObject();
+        }
+        mapping.endObject();
+        MapperService mapperService = createIndex("test").mapperService();
+        DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE);
+        assertThat(mapper.mappingSource().toString(), containsString("\"type\":\"string\""));
+        assertWarnings("dynamic template [my_template] has invalid content [{\"match_mapping_type\":\"string\",\"mapping\":{\"type\":" +
+            "\"string\"}}], caused by [No mapper found for type [string]]");
+    }
+
+    public void testIllegalDynamicTemplateUnknownAttribute() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder();
+        mapping.startObject();
+        {
+            mapping.startObject("type");
+            mapping.startArray("dynamic_templates");
+            {
+                mapping.startObject();
+                mapping.startObject("my_template");
+                mapping.field("match_mapping_type", "string");
+                mapping.startObject("mapping");
+                mapping.field("type", "keyword");
+                mapping.field("foo", "bar");
+                mapping.endObject();
+                mapping.endObject();
+                mapping.endObject();
+            }
+            mapping.endArray();
+            mapping.endObject();
+        }
+        mapping.endObject();
+        MapperService mapperService = createIndex("test").mapperService();
+        DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE);
+        assertThat(mapper.mappingSource().toString(), containsString("\"foo\":\"bar\""));
+        assertWarnings("dynamic template [my_template] has invalid content [{\"match_mapping_type\":\"string\",\"mapping\":{" +
+            "\"foo\":\"bar\",\"type\":\"keyword\"}}], caused by [Unused mapping attributes [{foo=bar}]]");
+    }
+
+    public void testIllegalDynamicTemplateInvalidAttribute() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder();
+        mapping.startObject();
+        {
+            mapping.startObject("type");
+            mapping.startArray("dynamic_templates");
+            {
+                mapping.startObject();
+                mapping.startObject("my_template");
+                mapping.field("match_mapping_type", "string");
+                mapping.startObject("mapping");
+                mapping.field("type", "text");
+                mapping.field("analyzer", "foobar");
+                mapping.endObject();
+                mapping.endObject();
+                mapping.endObject();
+            }
+            mapping.endArray();
+            mapping.endObject();
+        }
+        mapping.endObject();
+        MapperService mapperService = createIndex("test").mapperService();
+        DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE);
+        assertThat(mapper.mappingSource().toString(), containsString("\"analyzer\":\"foobar\""));
+        assertWarnings("dynamic template [my_template] has invalid content [{\"match_mapping_type\":\"string\",\"mapping\":{" +
+            "\"analyzer\":\"foobar\",\"type\":\"text\"}}], caused by [analyzer [foobar] not found for field [__dummy__]]");
+    }
+
+    public void testIllegalDynamicTemplateNoMappingType() throws Exception {
+        MapperService mapperService;
+
+        {
+            XContentBuilder mapping = XContentFactory.jsonBuilder();
+            mapping.startObject();
+            {
+                mapping.startObject("type");
+                mapping.startArray("dynamic_templates");
+                {
+                    mapping.startObject();
+                    mapping.startObject("my_template");
+                    if (randomBoolean()) {
+                        mapping.field("match_mapping_type", "*");
+                    } else {
+                        mapping.field("match", "string_*");
+                    }
+                    mapping.startObject("mapping");
+                    mapping.field("type", "{dynamic_type}");
+                    mapping.field("index_phrases", true);
+                    mapping.endObject();
+                    mapping.endObject();
+                    mapping.endObject();
+                }
+                mapping.endArray();
+                mapping.endObject();
+            }
+            mapping.endObject();
+            mapperService = createIndex("test").mapperService();
+            DocumentMapper mapper =
+                mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE);
+            assertThat(mapper.mappingSource().toString(), containsString("\"index_phrases\":true"));
+        }
+        {
+            boolean useMatchMappingType = randomBoolean();
+            XContentBuilder mapping = XContentFactory.jsonBuilder();
+            mapping.startObject();
+            {
+                mapping.startObject("type");
+                mapping.startArray("dynamic_templates");
+                {
+                    mapping.startObject();
+                    mapping.startObject("my_template");
+                    if (useMatchMappingType) {
+                        mapping.field("match_mapping_type", "*");
+                    } else {
+                        mapping.field("match", "string_*");
+                    }
+                    mapping.startObject("mapping");
+                    mapping.field("type", "{dynamic_type}");
+                    mapping.field("foo", "bar");
+                    mapping.endObject();
+                    mapping.endObject();
+                    mapping.endObject();
+                }
+                mapping.endArray();
+                mapping.endObject();
+            }
+            mapping.endObject();
+            DocumentMapper mapper =
+                mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE);
+            assertThat(mapper.mappingSource().toString(), containsString("\"foo\":\"bar\""));
+            if (useMatchMappingType) {
+                assertWarnings("dynamic template [my_template] has invalid content [{\"match_mapping_type\":\"*\",\"mapping\":{" +
+                    "\"foo\":\"bar\",\"type\":\"{dynamic_type}\"}}], caused by [Unused mapping attributes [{foo=bar}]]");
+            } else {
+                assertWarnings("dynamic template [my_template] has invalid content [{\"match\":\"string_*\",\"mapping\":{" +
+                    "\"foo\":\"bar\",\"type\":\"{dynamic_type}\"}}], caused by [Unused mapping attributes [{foo=bar}]]");
+            }
+        }
+    }
+
+    @Override
+    protected boolean forbidPrivateIndexSettings() {
+        return false;
+    }
+
+    public void testIllegalDynamicTemplatePre7Dot7Index() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder();
+        mapping.startObject();
+        {
+            mapping.startObject("type");
+            mapping.startArray("dynamic_templates");
+            {
+                mapping.startObject();
+                mapping.startObject("my_template");
+                mapping.field("match_mapping_type", "string");
+                mapping.startObject("mapping");
+                mapping.field("type", "string");
+                mapping.endObject();
+                mapping.endObject();
+                mapping.endObject();
+            }
+            mapping.endArray();
+            mapping.endObject();
+        }
+        mapping.endObject();
+        Version createdVersion = randomVersionBetween(random(), Version.V_7_0_0, Version.V_7_6_0);
+        Settings indexSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey(), createdVersion)
+            .build();
+        MapperService mapperService = createIndex("test", indexSettings).mapperService();
+        DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE);
+        assertThat(mapper.mappingSource().toString(), containsString("\"type\":\"string\""));
+    }
 }