Elasticsearch should reject dynamic templates with unknown `match_mapping_type`. #17285

When looking at the logstash template, I noticed that it has definitions for
dynamic temilates with `match_mapping_type` equal to `byte` for instance.
However elasticsearch never tries to find templates that match the byte type
(only long or double as far as numbers are concerned). This commit changes
template parsing in order to ignore bad values of `match_mapping_type` (given
how the logstash template is popular, this would break many upgrades
otherwise). Then I hope to fail the parsing on bad values in 6.0.
This commit is contained in:
Adrien Grand 2016-03-23 16:03:01 +01:00
parent 32a5e16c7d
commit 0854b03f13
4 changed files with 192 additions and 65 deletions

View File

@ -43,6 +43,7 @@ import org.elasticsearch.index.mapper.core.TextFieldMapper.TextFieldType;
import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import org.elasticsearch.index.mapper.object.ArrayValueMapperParser;
import org.elasticsearch.index.mapper.object.DynamicTemplate.XContentFieldType;
import org.elasticsearch.index.mapper.object.ObjectMapper;
import java.io.IOException;
@ -471,7 +472,7 @@ final class DocumentParser {
if (dynamic == ObjectMapper.Dynamic.STRICT) {
throw new StrictDynamicMappingException(mapper.fullPath(), currentFieldName);
} else if (dynamic == ObjectMapper.Dynamic.TRUE) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "object");
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.OBJECT);
if (builder == null) {
builder = new ObjectMapper.Builder(currentFieldName).enabled(true);
}
@ -516,7 +517,7 @@ final class DocumentParser {
if (dynamic == ObjectMapper.Dynamic.STRICT) {
throw new StrictDynamicMappingException(parentMapper.fullPath(), arrayFieldName);
} else if (dynamic == ObjectMapper.Dynamic.TRUE) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, arrayFieldName, "object");
Mapper.Builder builder = context.root().findTemplateBuilder(context, arrayFieldName, XContentFieldType.OBJECT);
if (builder == null) {
parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
} else {
@ -596,34 +597,34 @@ final class DocumentParser {
private static Mapper.Builder<?,?> createBuilderFromFieldType(final ParseContext context, MappedFieldType fieldType, String currentFieldName) {
Mapper.Builder builder = null;
if (fieldType instanceof StringFieldType) {
builder = context.root().findTemplateBuilder(context, currentFieldName, "string", "string");
builder = context.root().findTemplateBuilder(context, currentFieldName, "string", XContentFieldType.STRING);
} else if (fieldType instanceof TextFieldType) {
builder = context.root().findTemplateBuilder(context, currentFieldName, "text", "string");
builder = context.root().findTemplateBuilder(context, currentFieldName, "text", XContentFieldType.STRING);
if (builder == null) {
builder = new TextFieldMapper.Builder(currentFieldName)
.addMultiField(new KeywordFieldMapper.Builder("keyword").ignoreAbove(256));
}
} else if (fieldType instanceof KeywordFieldType) {
builder = context.root().findTemplateBuilder(context, currentFieldName, "keyword", "string");
builder = context.root().findTemplateBuilder(context, currentFieldName, "keyword", XContentFieldType.STRING);
} else {
switch (fieldType.typeName()) {
case DateFieldMapper.CONTENT_TYPE:
builder = context.root().findTemplateBuilder(context, currentFieldName, "date");
builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DATE);
break;
case "long":
builder = context.root().findTemplateBuilder(context, currentFieldName, "long");
builder = context.root().findTemplateBuilder(context, currentFieldName, "long", XContentFieldType.LONG);
break;
case "double":
builder = context.root().findTemplateBuilder(context, currentFieldName, "double");
builder = context.root().findTemplateBuilder(context, currentFieldName, "double", XContentFieldType.DOUBLE);
break;
case "integer":
builder = context.root().findTemplateBuilder(context, currentFieldName, "integer");
builder = context.root().findTemplateBuilder(context, currentFieldName, "integer", XContentFieldType.LONG);
break;
case "float":
builder = context.root().findTemplateBuilder(context, currentFieldName, "float");
builder = context.root().findTemplateBuilder(context, currentFieldName, "float", XContentFieldType.DOUBLE);
break;
case BooleanFieldMapper.CONTENT_TYPE:
builder = context.root().findTemplateBuilder(context, currentFieldName, "boolean");
builder = context.root().findTemplateBuilder(context, currentFieldName, "boolean", XContentFieldType.BOOLEAN);
break;
default:
break;
@ -682,7 +683,7 @@ final class DocumentParser {
for (FormatDateTimeFormatter dateTimeFormatter : context.root().dynamicDateTimeFormatters()) {
try {
dateTimeFormatter.parser().parseMillis(text);
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "date");
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DATE);
if (builder == null) {
builder = newDateBuilder(currentFieldName, dateTimeFormatter, Version.indexCreated(context.indexSettings()));
}
@ -697,7 +698,7 @@ final class DocumentParser {
String text = context.parser().text();
try {
Long.parseLong(text);
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long");
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG);
if (builder == null) {
builder = newLongBuilder(currentFieldName, Version.indexCreated(context.indexSettings()));
}
@ -707,7 +708,7 @@ final class DocumentParser {
}
try {
Double.parseDouble(text);
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double");
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE);
if (builder == null) {
builder = newFloatBuilder(currentFieldName, Version.indexCreated(context.indexSettings()));
}
@ -716,7 +717,7 @@ final class DocumentParser {
// not a long number
}
}
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "string");
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING);
if (builder == null) {
builder = new TextFieldMapper.Builder(currentFieldName)
.addMultiField(new KeywordFieldMapper.Builder("keyword").ignoreAbove(256));
@ -725,13 +726,13 @@ final class DocumentParser {
} else if (token == XContentParser.Token.VALUE_NUMBER) {
XContentParser.NumberType numberType = context.parser().numberType();
if (numberType == XContentParser.NumberType.INT || numberType == XContentParser.NumberType.LONG) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long");
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG);
if (builder == null) {
builder = newLongBuilder(currentFieldName, Version.indexCreated(context.indexSettings()));
}
return builder;
} else if (numberType == XContentParser.NumberType.FLOAT || numberType == XContentParser.NumberType.DOUBLE) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double");
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE);
if (builder == null) {
// no templates are defined, we use float by default instead of double
// since this is much more space-efficient and should be enough most of
@ -741,19 +742,19 @@ final class DocumentParser {
return builder;
}
} else if (token == XContentParser.Token.VALUE_BOOLEAN) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "boolean");
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.BOOLEAN);
if (builder == null) {
builder = new BooleanFieldMapper.Builder(currentFieldName);
}
return builder;
} else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "binary");
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.BINARY);
if (builder == null) {
builder = new BinaryFieldMapper.Builder(currentFieldName);
}
return builder;
} else {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, null);
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING);
if (builder != null) {
return builder;
}
@ -858,7 +859,7 @@ final class DocumentParser {
case STRICT:
throw new StrictDynamicMappingException(parent.fullPath(), paths[i]);
case TRUE:
Mapper.Builder builder = context.root().findTemplateBuilder(context, paths[i], "object");
Mapper.Builder builder = context.root().findTemplateBuilder(context, paths[i], XContentFieldType.OBJECT);
if (builder == null) {
builder = new ObjectMapper.Builder(paths[i]).enabled(true);
}

View File

@ -20,15 +20,21 @@
package org.elasticsearch.index.mapper.object;
import org.elasticsearch.Version;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.mapper.ContentPath;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.core.BinaryFieldMapper;
import org.elasticsearch.index.mapper.core.BooleanFieldMapper;
import org.elasticsearch.index.mapper.core.DateFieldMapper;
import org.elasticsearch.index.mapper.core.NumberFieldMapper;
import org.elasticsearch.index.mapper.core.TextFieldMapper;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -39,6 +45,8 @@ import java.util.TreeMap;
*/
public class DynamicTemplate implements ToXContent {
private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(DynamicTemplate.class));
public static enum MatchType {
SIMPLE {
@Override
@ -74,6 +82,93 @@ public class DynamicTemplate implements ToXContent {
public abstract boolean matches(String regex, String value);
}
/** The type of a field as detected while parsing a json document. */
public enum XContentFieldType {
OBJECT {
@Override
public String defaultMappingType() {
return ObjectMapper.CONTENT_TYPE;
}
@Override
public String toString() {
return "object";
}
},
STRING {
@Override
public String defaultMappingType() {
return TextFieldMapper.CONTENT_TYPE;
}
@Override
public String toString() {
return "string";
}
},
LONG {
@Override
public String defaultMappingType() {
return NumberFieldMapper.NumberType.LONG.typeName();
}
@Override
public String toString() {
return "long";
}
},
DOUBLE {
@Override
public String defaultMappingType() {
return NumberFieldMapper.NumberType.FLOAT.typeName();
}
@Override
public String toString() {
return "double";
}
},
BOOLEAN {
@Override
public String defaultMappingType() {
return BooleanFieldMapper.CONTENT_TYPE;
}
@Override
public String toString() {
return "boolean";
}
},
DATE {
@Override
public String defaultMappingType() {
return DateFieldMapper.CONTENT_TYPE;
}
@Override
public String toString() {
return "date";
}
},
BINARY {
@Override
public String defaultMappingType() {
return BinaryFieldMapper.CONTENT_TYPE;
}
@Override
public String toString() {
return "binary";
}
};
public static XContentFieldType fromString(String value) {
for (XContentFieldType v : values()) {
if (v.toString().equals(value)) {
return v;
}
}
throw new IllegalArgumentException("No xcontent type matched on [" + value + "], possible values are "
+ Arrays.toString(values()));
}
/** The default mapping type to use for fields of this {@link XContentFieldType}. */
public abstract String defaultMappingType();
}
public static DynamicTemplate parse(String name, Map<String, Object> conf,
Version indexVersionCreated) throws MapperParsingException {
String match = null;
@ -107,7 +202,30 @@ public class DynamicTemplate implements ToXContent {
}
}
return new DynamicTemplate(name, pathMatch, pathUnmatch, match, unmatch, matchMappingType, MatchType.fromString(matchPattern), mapping);
if (match == null && pathMatch == null && matchMappingType == null) {
throw new MapperParsingException("template must have match, path_match or match_mapping_type set " + conf.toString());
}
if (mapping == null) {
throw new MapperParsingException("template must have mapping set");
}
XContentFieldType xcontentFieldType = null;
if (matchMappingType != null && matchMappingType.equals("*") == false) {
try {
xcontentFieldType = XContentFieldType.fromString(matchMappingType);
} catch (IllegalArgumentException e) {
// TODO: do this in 6.0
/*if (indexVersionCreated.onOrAfter(Version.V_6_0_0)) {
throw e;
}*/
DEPRECATION_LOGGER.deprecated("Ignoring unrecognized match_mapping_type: [" + matchMappingType + "]");
// this template is on an unknown type so it will never match anything
// null indicates that the template should be ignored
return null;
}
}
return new DynamicTemplate(name, pathMatch, pathUnmatch, match, unmatch, xcontentFieldType, MatchType.fromString(matchPattern), mapping);
}
private final String name;
@ -122,24 +240,19 @@ public class DynamicTemplate implements ToXContent {
private final MatchType matchType;
private final String matchMappingType;
private final XContentFieldType xcontentFieldType;
private final Map<String, Object> mapping;
public DynamicTemplate(String name, String pathMatch, String pathUnmatch, String match, String unmatch, String matchMappingType, MatchType matchType, Map<String, Object> mapping) {
if (match == null && pathMatch == null && matchMappingType == null) {
throw new MapperParsingException("template must have match, path_match or match_mapping_type set");
}
if (mapping == null) {
throw new MapperParsingException("template must have mapping set");
}
private DynamicTemplate(String name, String pathMatch, String pathUnmatch, String match, String unmatch,
XContentFieldType xcontentFieldType, MatchType matchType, Map<String, Object> mapping) {
this.name = name;
this.pathMatch = pathMatch;
this.pathUnmatch = pathUnmatch;
this.match = match;
this.unmatch = unmatch;
this.matchType = matchType;
this.matchMappingType = matchMappingType;
this.xcontentFieldType = xcontentFieldType;
this.mapping = mapping;
}
@ -147,26 +260,21 @@ public class DynamicTemplate implements ToXContent {
return this.name;
}
public boolean match(ContentPath path, String name, String dynamicType) {
if (pathMatch != null && !matchType.matches(pathMatch, path.pathAsText(name))) {
public boolean match(String path, String name, XContentFieldType xcontentFieldType) {
if (pathMatch != null && !matchType.matches(pathMatch, path)) {
return false;
}
if (match != null && !matchType.matches(match, name)) {
return false;
}
if (pathUnmatch != null && matchType.matches(pathUnmatch, path.pathAsText(name))) {
if (pathUnmatch != null && matchType.matches(pathUnmatch, path)) {
return false;
}
if (unmatch != null && matchType.matches(unmatch, name)) {
return false;
}
if (matchMappingType != null) {
if (dynamicType == null) {
return false;
}
if (!matchType.matches(matchMappingType, dynamicType)) {
return false;
}
if (this.xcontentFieldType != null && this.xcontentFieldType != xcontentFieldType) {
return false;
}
return true;
}
@ -248,8 +356,10 @@ public class DynamicTemplate implements ToXContent {
if (pathUnmatch != null) {
builder.field("path_unmatch", pathUnmatch);
}
if (matchMappingType != null) {
builder.field("match_mapping_type", matchMappingType);
if (xcontentFieldType != null) {
builder.field("match_mapping_type", xcontentFieldType);
} else if (match == null && pathMatch == null) {
builder.field("match_mapping_type", "*");
}
if (matchType != MatchType.SIMPLE) {
builder.field("match_pattern", matchType);

View File

@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper.object;
import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.joda.FormatDateTimeFormatter;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.settings.Settings;
@ -33,6 +32,7 @@ import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.core.DateFieldMapper;
import org.elasticsearch.index.mapper.object.DynamicTemplate.XContentFieldType;
import java.io.IOException;
import java.util.ArrayList;
@ -190,7 +190,9 @@ public class RootObjectMapper extends ObjectMapper {
String templateName = entry.getKey();
Map<String, Object> templateParams = (Map<String, Object>) entry.getValue();
DynamicTemplate template = DynamicTemplate.parse(templateName, templateParams, indexVersionCreated);
((Builder) builder).add(template);
if (template != null) {
((Builder) builder).add(template);
}
}
return true;
} else if (fieldName.equals("date_detection")) {
@ -240,21 +242,8 @@ public class RootObjectMapper extends ObjectMapper {
return dynamicDateTimeFormatters;
}
public Mapper.Builder findTemplateBuilder(ParseContext context, String name, String matchType) {
final String dynamicType;
switch (matchType) {
case "string":
// string is a corner case since a json string can either map to a
// text or keyword field in elasticsearch. For now we use text when
// unspecified. For other types, the mapping type matches the json
// type so we are fine
dynamicType = "text";
break;
default:
dynamicType = matchType;
break;
}
return findTemplateBuilder(context, name, dynamicType, matchType);
public Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType) {
return findTemplateBuilder(context, name, matchType.defaultMappingType(), matchType);
}
/**
@ -264,7 +253,7 @@ public class RootObjectMapper extends ObjectMapper {
* @param matchType the type of the field in the json document or null if unknown
* @return a mapper builder, or null if there is no template for such a field
*/
public Mapper.Builder findTemplateBuilder(ParseContext context, String name, String dynamicType, String matchType) {
public Mapper.Builder findTemplateBuilder(ParseContext context, String name, String dynamicType, XContentFieldType matchType) {
DynamicTemplate dynamicTemplate = findTemplate(context.path(), name, matchType);
if (dynamicTemplate == null) {
return null;
@ -278,9 +267,10 @@ public class RootObjectMapper extends ObjectMapper {
return typeParser.parse(name, dynamicTemplate.mappingForName(name, dynamicType), parserContext);
}
private DynamicTemplate findTemplate(ContentPath path, String name, String matchType) {
public DynamicTemplate findTemplate(ContentPath path, String name, XContentFieldType matchType) {
final String pathAsString = path.pathAsText(name);
for (DynamicTemplate dynamicTemplate : dynamicTemplates) {
if (dynamicTemplate.match(path, name, matchType)) {
if (dynamicTemplate.match(pathAsString, name, matchType)) {
return dynamicTemplate;
}
}

View File

@ -24,6 +24,7 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.mapper.object.DynamicTemplate;
import org.elasticsearch.index.mapper.object.DynamicTemplate.XContentFieldType;
import org.elasticsearch.test.ESTestCase;
import java.util.Collections;
@ -49,6 +50,31 @@ public class DynamicTemplateTests extends ESTestCase {
assertEquals("{\"match_mapping_type\":\"string\",\"mapping\":{\"store\":true}}", builder.string());
}
public void testParseUnknownMatchType() {
Map<String, Object> templateDef = new HashMap<>();
templateDef.put("match_mapping_type", "short");
templateDef.put("mapping", Collections.singletonMap("store", true));
// if a wrong match type is specified, we ignore the template
assertNull(DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha5));
}
public void testMatchAllTemplate() {
Map<String, Object> templateDef = new HashMap<>();
templateDef.put("match_mapping_type", "*");
templateDef.put("mapping", Collections.singletonMap("store", true));
DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha5);
assertTrue(template.match("a.b", "b", randomFrom(XContentFieldType.values())));
}
public void testMatchTypeTemplate() {
Map<String, Object> templateDef = new HashMap<>();
templateDef.put("match_mapping_type", "string");
templateDef.put("mapping", Collections.singletonMap("store", true));
DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha5);
assertTrue(template.match("a.b", "b", XContentFieldType.STRING));
assertFalse(template.match("a.b", "b", XContentFieldType.BOOLEAN));
}
public void testSerialization() throws Exception {
// type-based template
Map<String, Object> templateDef = new HashMap<>();