honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Mappings: Support dots in field names when mapping exists 

In 2.0 we began restricting fields to not contains dots in their names.
This change adds back part of dots in fieldnames support. Specifically,
it allows indexing documents that contain dots in the field names, when
the correct corresponding mappers exist. For example, if mappings
contain an object field `foo`, and a subfield `bar`, then indexing a
document with `foo.bar` will work.

see #15951
This commit is contained in:
Ryan Ernst 2016-04-14 11:31:45 -07:00
parent f35cfc3715
commit 125473dc9f
2 changed files with 58 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
View File

@ -54,7 +54,6 @@ import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import org.elasticsearch.index.mapper.object.ArrayValueMapperParser; import org.elasticsearch.index.mapper.object.ArrayValueMapperParser;
import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.mapper.object.ObjectMapper;
import org.elasticsearch.index.mapper.object.RootObjectMapper;
/** A parser for documents, given mappings from a DocumentMapper */ /** A parser for documents, given mappings from a DocumentMapper */
final class DocumentParser implements Closeable { final class DocumentParser implements Closeable {
@ -474,7 +473,7 @@ final class DocumentParser implements Closeable {
context.addDynamicMapper(update); context.addDynamicMapper(update);
} }
if (fieldMapper.copyTo() != null) { if (fieldMapper.copyTo() != null) {
parseCopyFields(context, fieldMapper, fieldMapper.copyTo().copyToFields()); parseCopyFields(context, fieldMapper.copyTo().copyToFields());
} }
} }
} }
@ -484,14 +483,11 @@ final class DocumentParser implements Closeable {
context.path().add(currentFieldName); context.path().add(currentFieldName);
ObjectMapper update = null; ObjectMapper update = null;
Mapper objectMapper = mapper.getMapper(currentFieldName); Mapper objectMapper = getMapper(mapper, currentFieldName);
if (objectMapper != null) { if (objectMapper != null) {
parseObjectOrField(context, objectMapper); parseObjectOrField(context, objectMapper);
} else { } else {
ObjectMapper.Dynamic dynamic = mapper.dynamic(); ObjectMapper.Dynamic dynamic = dynamicOrDefault(mapper, context.root().dynamic());
if (dynamic == null) {
dynamic = dynamicOrDefault(context.root().dynamic());
}
if (dynamic == ObjectMapper.Dynamic.STRICT) { if (dynamic == ObjectMapper.Dynamic.STRICT) {
throw new StrictDynamicMappingException(mapper.fullPath(), currentFieldName); throw new StrictDynamicMappingException(mapper.fullPath(), currentFieldName);
} else if (dynamic == ObjectMapper.Dynamic.TRUE) { } else if (dynamic == ObjectMapper.Dynamic.TRUE) {
@ -500,10 +496,6 @@ final class DocumentParser implements Closeable {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "object"); Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "object");
if (builder == null) { if (builder == null) {
builder = new ObjectMapper.Builder(currentFieldName).enabled(true); builder = new ObjectMapper.Builder(currentFieldName).enabled(true);
// if this is a non root object, then explicitly set the dynamic behavior if set
if (!(mapper instanceof RootObjectMapper) && mapper.dynamic() != ObjectMapper.Defaults.DYNAMIC) {
((ObjectMapper.Builder) builder).dynamic(mapper.dynamic());
}
} }
Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path()); Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path());
objectMapper = builder.build(builderContext); objectMapper = builder.build(builderContext);
@ -522,7 +514,7 @@ final class DocumentParser implements Closeable {
private static void parseArray(ParseContext context, ObjectMapper parentMapper, String lastFieldName) throws IOException { private static void parseArray(ParseContext context, ObjectMapper parentMapper, String lastFieldName) throws IOException {
String arrayFieldName = lastFieldName; String arrayFieldName = lastFieldName;
Mapper mapper = parentMapper.getMapper(lastFieldName); Mapper mapper = getMapper(parentMapper, lastFieldName);
if (mapper != null) { if (mapper != null) {
// There is a concrete mapper for this field already. Need to check if the mapper // There is a concrete mapper for this field already. Need to check if the mapper
// expects an array, if so we pass the context straight to the mapper and if not // expects an array, if so we pass the context straight to the mapper and if not
@ -534,10 +526,7 @@ final class DocumentParser implements Closeable {
} }
} else { } else {
ObjectMapper.Dynamic dynamic = parentMapper.dynamic(); ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context.root().dynamic());
if (dynamic == null) {
dynamic = dynamicOrDefault(context.root().dynamic());
}
if (dynamic == ObjectMapper.Dynamic.STRICT) { if (dynamic == ObjectMapper.Dynamic.STRICT) {
throw new StrictDynamicMappingException(parentMapper.fullPath(), arrayFieldName); throw new StrictDynamicMappingException(parentMapper.fullPath(), arrayFieldName);
} else if (dynamic == ObjectMapper.Dynamic.TRUE) { } else if (dynamic == ObjectMapper.Dynamic.TRUE) {
@ -587,7 +576,7 @@ final class DocumentParser implements Closeable {
if (currentFieldName == null) { if (currentFieldName == null) {
throw new MapperParsingException("object mapping [" + parentMapper.name() + "] trying to serialize a value with no field associated with it, current value [" + context.parser().textOrNull() + "]"); throw new MapperParsingException("object mapping [" + parentMapper.name() + "] trying to serialize a value with no field associated with it, current value [" + context.parser().textOrNull() + "]");
} }
Mapper mapper = parentMapper.getMapper(currentFieldName); Mapper mapper = getMapper(parentMapper, currentFieldName);
if (mapper != null) { if (mapper != null) {
parseObjectOrField(context, mapper); parseObjectOrField(context, mapper);
} else { } else {
@ -597,7 +586,7 @@ final class DocumentParser implements Closeable {
private static void parseNullValue(ParseContext context, ObjectMapper parentMapper, String lastFieldName) throws IOException { private static void parseNullValue(ParseContext context, ObjectMapper parentMapper, String lastFieldName) throws IOException {
// we can only handle null values if we have mappings for them // we can only handle null values if we have mappings for them
Mapper mapper = parentMapper.getMapper(lastFieldName); Mapper mapper = getMapper(parentMapper, lastFieldName);
if (mapper != null) { if (mapper != null) {
// TODO: passing null to an object seems bogus? // TODO: passing null to an object seems bogus?
parseObjectOrField(context, mapper); parseObjectOrField(context, mapper);
@ -811,10 +800,7 @@ final class DocumentParser implements Closeable {
} }
private static void parseDynamicValue(final ParseContext context, ObjectMapper parentMapper, String currentFieldName, XContentParser.Token token) throws IOException { private static void parseDynamicValue(final ParseContext context, ObjectMapper parentMapper, String currentFieldName, XContentParser.Token token) throws IOException {
ObjectMapper.Dynamic dynamic = parentMapper.dynamic(); ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context.root().dynamic());
if (dynamic == null) {
dynamic = dynamicOrDefault(context.root().dynamic());
}
if (dynamic == ObjectMapper.Dynamic.STRICT) { if (dynamic == ObjectMapper.Dynamic.STRICT) {
throw new StrictDynamicMappingException(parentMapper.fullPath(), currentFieldName); throw new StrictDynamicMappingException(parentMapper.fullPath(), currentFieldName);
} }
@ -824,12 +810,11 @@ final class DocumentParser implements Closeable {
final String path = context.path().pathAsText(currentFieldName); final String path = context.path().pathAsText(currentFieldName);
final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path()); final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path());
final MappedFieldType existingFieldType = context.mapperService().fullName(path); final MappedFieldType existingFieldType = context.mapperService().fullName(path);
Mapper.Builder builder = null; final Mapper.Builder builder;
if (existingFieldType != null) { if (existingFieldType != null) {
// create a builder of the same type // create a builder of the same type
builder = createBuilderFromFieldType(context, existingFieldType, currentFieldName); builder = createBuilderFromFieldType(context, existingFieldType, currentFieldName);
} } else {
if (builder == null) {
builder = createBuilderFromDynamicValue(context, token, currentFieldName); builder = createBuilderFromDynamicValue(context, token, currentFieldName);
} }
Mapper mapper = builder.build(builderContext); Mapper mapper = builder.build(builderContext);
@ -843,7 +828,7 @@ final class DocumentParser implements Closeable {
} }
/** Creates instances of the fields that the current field should be copied to */ /** Creates instances of the fields that the current field should be copied to */
private static void parseCopyFields(ParseContext context, FieldMapper fieldMapper, List<String> copyToFields) throws IOException { private static void parseCopyFields(ParseContext context, List<String> copyToFields) throws IOException {
if (!context.isWithinCopyTo() && copyToFields.isEmpty() == false) { if (!context.isWithinCopyTo() && copyToFields.isEmpty() == false) {
context = context.createCopyToContext(); context = context.createCopyToContext();
for (String field : copyToFields) { for (String field : copyToFields) {
@ -888,10 +873,7 @@ final class DocumentParser implements Closeable {
mapper = context.docMapper().objectMappers().get(context.path().pathAsText(paths[i])); mapper = context.docMapper().objectMappers().get(context.path().pathAsText(paths[i]));
if (mapper == null) { if (mapper == null) {
// One mapping is missing, check if we are allowed to create a dynamic one. // One mapping is missing, check if we are allowed to create a dynamic one.
ObjectMapper.Dynamic dynamic = parent.dynamic(); ObjectMapper.Dynamic dynamic = dynamicOrDefault(parent, context.root().dynamic());
if (dynamic == null) {
dynamic = dynamicOrDefault(context.root().dynamic());
}
switch (dynamic) { switch (dynamic) {
case STRICT: case STRICT:
@ -899,10 +881,6 @@ final class DocumentParser implements Closeable {
case TRUE: case TRUE:
Mapper.Builder builder = context.root().findTemplateBuilder(context, paths[i], "object"); Mapper.Builder builder = context.root().findTemplateBuilder(context, paths[i], "object");
if (builder == null) { if (builder == null) {
// if this is a non root object, then explicitly set the dynamic behavior if set
if (!(parent instanceof RootObjectMapper) && parent.dynamic() != ObjectMapper.Defaults.DYNAMIC) {
((ObjectMapper.Builder) builder).dynamic(parent.dynamic());
}
builder = new ObjectMapper.Builder(paths[i]).enabled(true); builder = new ObjectMapper.Builder(paths[i]).enabled(true);
} }
Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path()); Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path());
@ -915,8 +893,6 @@ final class DocumentParser implements Closeable {
case FALSE: case FALSE:
// Maybe we should log something to tell the user that the copy_to is ignored in this case. // Maybe we should log something to tell the user that the copy_to is ignored in this case.
break; break;
default:
throw new AssertionError("Unexpected dynamic type " + dynamic);
} }
} }
@ -929,8 +905,25 @@ final class DocumentParser implements Closeable {
} }
} }
private static ObjectMapper.Dynamic dynamicOrDefault(ObjectMapper.Dynamic dynamic) { private static ObjectMapper.Dynamic dynamicOrDefault(ObjectMapper parentMapper, ObjectMapper.Dynamic dynamicDefault) {
return dynamic == null ? ObjectMapper.Dynamic.TRUE : dynamic; ObjectMapper.Dynamic dynamic = parentMapper.dynamic();
if (dynamic == null) {
return dynamicDefault == null ? ObjectMapper.Dynamic.TRUE : dynamicDefault;
}
return dynamic;
}
// looks up a child mapper, but takes into account field names that expand to objects
static Mapper getMapper(ObjectMapper objectMapper, String fieldName) {
String[] subfields = fieldName.split("\\.");
for (int i = 0; i < subfields.length - 1; ++i) {
Mapper mapper = objectMapper.getMapper(subfields[i]);
if (mapper == null || (mapper instanceof ObjectMapper) == false) {
return null;
}
objectMapper = (ObjectMapper)mapper;
}
return objectMapper.getMapper(subfields[subfields.length - 1]);
} }
@Override @Override

27
core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
View File

@ -70,6 +70,33 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
assertNotNull(doc.rootDoc().getField(UidFieldMapper.NAME)); assertNotNull(doc.rootDoc().getField(UidFieldMapper.NAME));
} }
public void testDotsWithExistingMapper() throws Exception {
DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("foo").startObject("properties")
.startObject("bar").startObject("properties")
.startObject("baz").field("type", "integer")
.endObject().endObject().endObject().endObject().endObject().endObject().endObject().endObject().string();
DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
BytesReference bytes = XContentFactory.jsonBuilder()
.startObject()
.field("foo.bar.baz", 123)
.startObject("foo")
.field("bar.baz", 456)
.endObject()
.startObject("foo.bar")
.field("baz", 789)
.endObject()
.endObject().bytes();
ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
String[] values = doc.rootDoc().getValues("foo.bar.baz");
assertEquals(3, values.length);
assertEquals("123", values[0]);
assertEquals("456", values[1]);
assertEquals("789", values[2]);
}
DocumentMapper createDummyMapping(MapperService mapperService) throws Exception { DocumentMapper createDummyMapping(MapperService mapperService) throws Exception {
String mapping = jsonBuilder().startObject().startObject("type").startObject("properties") String mapping = jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("y").field("type", "object").endObject() .startObject("y").field("type", "object").endObject()