honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

parent/child: Split the _parent field mapping's field type into three field types: 

1) A shared immutable fieldtype for the _parent field (used for direct access to that field in the dsl). This field type is stored and indexed.
2) A per type field type for the child join field. The field type has doc values enabled if index is created on or post 2.0 and field data type is allowed to be changed.
3) A per type field type for the parent join field. The field type has doc values enabled if index is created on or post 2.0.

This resolves the issue that a mapping is not compatible if parent and child types have different field data loading settings.

Closes #13169
This commit is contained in:
Martijn van Groningen 2015-09-08 14:47:05 +02:00
parent e40409dd7f
commit e23d116bc5
12 changed files with 326 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
View File

@ -113,7 +113,7 @@ public class DocumentMapper implements ToXContent {
this.rootMappers.put(TimestampFieldMapper.class, new TimestampFieldMapper(indexSettings, mapperService.fullName(TimestampFieldMapper.NAME))); this.rootMappers.put(TimestampFieldMapper.class, new TimestampFieldMapper(indexSettings, mapperService.fullName(TimestampFieldMapper.NAME)));
this.rootMappers.put(TTLFieldMapper.class, new TTLFieldMapper(indexSettings)); this.rootMappers.put(TTLFieldMapper.class, new TTLFieldMapper(indexSettings));
this.rootMappers.put(VersionFieldMapper.class, new VersionFieldMapper(indexSettings)); this.rootMappers.put(VersionFieldMapper.class, new VersionFieldMapper(indexSettings));
this.rootMappers.put(ParentFieldMapper.class, new ParentFieldMapper(indexSettings, mapperService.fullName(ParentFieldMapper.NAME))); this.rootMappers.put(ParentFieldMapper.class, new ParentFieldMapper(indexSettings, mapperService.fullName(ParentFieldMapper.NAME), /* parent type */builder.name()));
// _field_names last so that it can see all other fields // _field_names last so that it can see all other fields
this.rootMappers.put(FieldNamesFieldMapper.class, new FieldNamesFieldMapper(indexSettings, mapperService.fullName(FieldNamesFieldMapper.NAME))); this.rootMappers.put(FieldNamesFieldMapper.class, new FieldNamesFieldMapper(indexSettings, mapperService.fullName(FieldNamesFieldMapper.NAME)));
} }

6
core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
View File

@ -147,8 +147,8 @@ public class DocumentMapperParser {
} }
} }
public Mapper.TypeParser.ParserContext parserContext() { public Mapper.TypeParser.ParserContext parserContext(String type) {
return new Mapper.TypeParser.ParserContext(analysisService, similarityLookupService, mapperService, typeParsers, indexVersionCreated, parseFieldMatcher); return new Mapper.TypeParser.ParserContext(type, analysisService, similarityLookupService, mapperService, typeParsers, indexVersionCreated, parseFieldMatcher);
} }
public DocumentMapper parse(String source) throws MapperParsingException { public DocumentMapper parse(String source) throws MapperParsingException {
@ -206,7 +206,7 @@ public class DocumentMapperParser {
} }
Mapper.TypeParser.ParserContext parserContext = parserContext(); Mapper.TypeParser.ParserContext parserContext = parserContext(type);
// parse RootObjectMapper // parse RootObjectMapper
DocumentMapper.Builder docBuilder = doc(indexSettings, (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext), mapperService); DocumentMapper.Builder docBuilder = doc(indexSettings, (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext), mapperService);
// Add default mapping for the plugged-in meta mappers // Add default mapping for the plugged-in meta mappers

11
core/src/main/java/org/elasticsearch/index/mapper/Mapper.java
View File

@ -81,6 +81,8 @@ public abstract class Mapper implements ToXContent, Iterable<Mapper> {
class ParserContext { class ParserContext {
private final String type;
private final AnalysisService analysisService; private final AnalysisService analysisService;
private final SimilarityLookupService similarityLookupService; private final SimilarityLookupService similarityLookupService;
@ -93,9 +95,10 @@ public abstract class Mapper implements ToXContent, Iterable<Mapper> {
private final ParseFieldMatcher parseFieldMatcher; private final ParseFieldMatcher parseFieldMatcher;
public ParserContext(AnalysisService analysisService, SimilarityLookupService similarityLookupService, public ParserContext(String type, AnalysisService analysisService, SimilarityLookupService similarityLookupService,
MapperService mapperService, ImmutableMap<String, TypeParser> typeParsers, MapperService mapperService, ImmutableMap<String, TypeParser> typeParsers,
Version indexVersionCreated, ParseFieldMatcher parseFieldMatcher) { Version indexVersionCreated, ParseFieldMatcher parseFieldMatcher) {
this.type = type;
this.analysisService = analysisService; this.analysisService = analysisService;
this.similarityLookupService = similarityLookupService; this.similarityLookupService = similarityLookupService;
this.mapperService = mapperService; this.mapperService = mapperService;
@ -104,6 +107,10 @@ public abstract class Mapper implements ToXContent, Iterable<Mapper> {
this.parseFieldMatcher = parseFieldMatcher; this.parseFieldMatcher = parseFieldMatcher;
} }
public String type() {
return type;
}
public AnalysisService analysisService() { public AnalysisService analysisService() {
return analysisService; return analysisService;
} }

3
core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
View File

@ -70,7 +70,6 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Predicate;
import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder; import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder;
@ -556,7 +555,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
final ImmutableMap<String, MappedFieldType> unmappedFieldMappers = this.unmappedFieldTypes; final ImmutableMap<String, MappedFieldType> unmappedFieldMappers = this.unmappedFieldTypes;
MappedFieldType fieldType = unmappedFieldMappers.get(type); MappedFieldType fieldType = unmappedFieldMappers.get(type);
if (fieldType == null) { if (fieldType == null) {
final Mapper.TypeParser.ParserContext parserContext = documentMapperParser().parserContext(); final Mapper.TypeParser.ParserContext parserContext = documentMapperParser().parserContext(type);
Mapper.TypeParser typeParser = parserContext.typeParser(type); Mapper.TypeParser typeParser = parserContext.typeParser(type);
if (typeParser == null) { if (typeParser == null) {
throw new IllegalArgumentException("No mapper found for type [" + type + "]"); throw new IllegalArgumentException("No mapper found for type [" + type + "]");

156
core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
View File

@ -20,6 +20,7 @@ package org.elasticsearch.index.mapper.internal;
import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.queries.TermsQuery; import org.apache.lucene.queries.TermsQuery;
import org.apache.lucene.search.Query; import org.apache.lucene.search.Query;
@ -33,15 +34,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.loader.SettingsLoader; import org.elasticsearch.common.settings.loader.SettingsLoader;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.mapper.MergeResult;
import org.elasticsearch.index.mapper.MetadataFieldMapper;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryParseContext;
import java.io.IOException; import java.io.IOException;
@ -67,6 +60,7 @@ public class ParentFieldMapper extends MetadataFieldMapper {
public static final String NAME = ParentFieldMapper.NAME; public static final String NAME = ParentFieldMapper.NAME;
public static final MappedFieldType FIELD_TYPE = new ParentFieldType(); public static final MappedFieldType FIELD_TYPE = new ParentFieldType();
public static final MappedFieldType JOIN_FIELD_TYPE = new ParentFieldType();
static { static {
FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
@ -77,41 +71,66 @@ public class ParentFieldMapper extends MetadataFieldMapper {
FIELD_TYPE.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER); FIELD_TYPE.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
FIELD_TYPE.setNames(new MappedFieldType.Names(NAME)); FIELD_TYPE.setNames(new MappedFieldType.Names(NAME));
FIELD_TYPE.freeze(); FIELD_TYPE.freeze();
JOIN_FIELD_TYPE.setHasDocValues(true);
JOIN_FIELD_TYPE.setDocValuesType(DocValuesType.SORTED);
JOIN_FIELD_TYPE.freeze();
} }
} }
public static class Builder extends MetadataFieldMapper.Builder<Builder, ParentFieldMapper> { public static class Builder extends MetadataFieldMapper.Builder<Builder, ParentFieldMapper> {
private String parentType;
protected String indexName; protected String indexName;
private String type; private final String documentType;
public Builder() { private final MappedFieldType parentJoinFieldType = Defaults.JOIN_FIELD_TYPE.clone();
private final MappedFieldType childJoinFieldType = Defaults.JOIN_FIELD_TYPE.clone();
public Builder(String documentType) {
super(Defaults.NAME, Defaults.FIELD_TYPE); super(Defaults.NAME, Defaults.FIELD_TYPE);
this.indexName = name; this.indexName = name;
this.documentType = documentType;
builder = this; builder = this;
} }
public Builder type(String type) { public Builder type(String type) {
this.type = type; this.parentType = type;
return builder; return builder;
} }
@Override
public Builder fieldDataSettings(Settings fieldDataSettings) {
Settings settings = Settings.builder().put(childJoinFieldType.fieldDataType().getSettings()).put(fieldDataSettings).build();
childJoinFieldType.setFieldDataType(new FieldDataType(childJoinFieldType.fieldDataType().getType(), settings));
return this;
}
@Override @Override
public ParentFieldMapper build(BuilderContext context) { public ParentFieldMapper build(BuilderContext context) {
if (type == null) { if (parentType == null) {
throw new MapperParsingException("[_parent] field mapping must contain the [type] option"); throw new MapperParsingException("[_parent] field mapping must contain the [type] option");
} }
setupFieldType(context); parentJoinFieldType.setNames(new MappedFieldType.Names(joinField(documentType)));
fieldType.setHasDocValues(context.indexCreatedVersion().onOrAfter(Version.V_2_0_0_beta1)); parentJoinFieldType.setFieldDataType(null);
return new ParentFieldMapper(fieldType, type, context.indexSettings()); childJoinFieldType.setNames(new MappedFieldType.Names(joinField(parentType)));
if (context.indexCreatedVersion().before(Version.V_2_0_0_beta1)) {
childJoinFieldType.setHasDocValues(false);
childJoinFieldType.setDocValuesType(DocValuesType.NONE);
parentJoinFieldType.setHasDocValues(false);
parentJoinFieldType.setDocValuesType(DocValuesType.NONE);
}
return new ParentFieldMapper(fieldType, parentJoinFieldType, childJoinFieldType, parentType, context.indexSettings());
} }
} }
public static class TypeParser implements Mapper.TypeParser { public static class TypeParser implements Mapper.TypeParser {
@Override @Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException { public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
Builder builder = new Builder(); Builder builder = new Builder(parserContext.type());
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) { for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
Map.Entry<String, Object> entry = iterator.next(); Map.Entry<String, Object> entry = iterator.next();
String fieldName = Strings.toUnderscoreCase(entry.getKey()); String fieldName = Strings.toUnderscoreCase(entry.getKey());
@ -222,25 +241,50 @@ public class ParentFieldMapper extends MetadataFieldMapper {
} }
} }
private final String type; private final String parentType;
// determines the field data settings
private MappedFieldType childJoinFieldType;
// has no impact of field data settings, is just here for creating a join field, the parent field mapper in the child type pointing to this type determines the field data settings for this join field
private final MappedFieldType parentJoinFieldType;
protected ParentFieldMapper(MappedFieldType fieldType, String type, Settings indexSettings) { protected ParentFieldMapper(MappedFieldType fieldType, MappedFieldType parentJoinFieldType, MappedFieldType childJoinFieldType, String parentType, Settings indexSettings) {
super(NAME, setupDocValues(indexSettings, fieldType), setupDocValues(indexSettings, Defaults.FIELD_TYPE), indexSettings); super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings);
this.type = type; this.parentType = parentType;
this.parentJoinFieldType = parentJoinFieldType;
this.parentJoinFieldType.freeze();
this.childJoinFieldType = childJoinFieldType;
if (childJoinFieldType != null) {
this.childJoinFieldType.freeze();
}
} }
public ParentFieldMapper(Settings indexSettings, MappedFieldType existing) { public ParentFieldMapper(Settings indexSettings, MappedFieldType existing, String parentType) {
this(existing == null ? Defaults.FIELD_TYPE.clone() : existing.clone(), null, indexSettings); this(existing == null ? Defaults.FIELD_TYPE.clone() : existing.clone(), joinFieldTypeForParentType(parentType, indexSettings), null, null, indexSettings);
} }
static MappedFieldType setupDocValues(Settings indexSettings, MappedFieldType fieldType) { private static MappedFieldType joinFieldTypeForParentType(String parentType, Settings indexSettings) {
fieldType = fieldType.clone(); MappedFieldType parentJoinFieldType = Defaults.JOIN_FIELD_TYPE.clone();
fieldType.setHasDocValues(Version.indexCreated(indexSettings).onOrAfter(Version.V_2_0_0_beta1)); parentJoinFieldType.setNames(new MappedFieldType.Names(joinField(parentType)));
return fieldType;
Version indexCreated = Version.indexCreated(indexSettings);
if (indexCreated.before(Version.V_2_0_0_beta1)) {
parentJoinFieldType.setHasDocValues(false);
parentJoinFieldType.setDocValuesType(DocValuesType.NONE);
}
parentJoinFieldType.freeze();
return parentJoinFieldType;
}
public MappedFieldType getParentJoinFieldType() {
return parentJoinFieldType;
}
public MappedFieldType getChildJoinFieldType() {
return childJoinFieldType;
} }
public String type() { public String type() {
return type; return parentType;
} }
@Override @Override
@ -257,8 +301,8 @@ public class ParentFieldMapper extends MetadataFieldMapper {
@Override @Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException { protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
boolean parent = context.docMapper().isParent(context.type()); boolean parent = context.docMapper().isParent(context.type());
if (parent && fieldType().hasDocValues()) { if (parent) {
fields.add(createJoinField(context.type(), context.id())); addJoinFieldIfNeeded(fields, parentJoinFieldType, context.id());
} }
if (!active()) { if (!active()) {
@ -269,10 +313,8 @@ public class ParentFieldMapper extends MetadataFieldMapper {
// we are in the parsing of _parent phase // we are in the parsing of _parent phase
String parentId = context.parser().text(); String parentId = context.parser().text();
context.sourceToParse().parent(parentId); context.sourceToParse().parent(parentId);
fields.add(new Field(fieldType().names().indexName(), Uid.createUid(context.stringBuilder(), type, parentId), fieldType())); fields.add(new Field(fieldType().names().indexName(), Uid.createUid(context.stringBuilder(), parentType, parentId), fieldType()));
if (fieldType().hasDocValues()) { addJoinFieldIfNeeded(fields, childJoinFieldType, parentId);
fields.add(createJoinField(type, parentId));
}
} else { } else {
// otherwise, we are running it post processing of the xcontent // otherwise, we are running it post processing of the xcontent
String parsedParentId = context.doc().get(Defaults.NAME); String parsedParentId = context.doc().get(Defaults.NAME);
@ -283,11 +325,9 @@ public class ParentFieldMapper extends MetadataFieldMapper {
throw new MapperParsingException("No parent id provided, not within the document, and not externally"); throw new MapperParsingException("No parent id provided, not within the document, and not externally");
} }
// we did not add it in the parsing phase, add it now // we did not add it in the parsing phase, add it now
fields.add(new Field(fieldType().names().indexName(), Uid.createUid(context.stringBuilder(), type, parentId), fieldType())); fields.add(new Field(fieldType().names().indexName(), Uid.createUid(context.stringBuilder(), parentType, parentId), fieldType()));
if (fieldType().hasDocValues()) { addJoinFieldIfNeeded(fields, childJoinFieldType, parentId);
fields.add(createJoinField(type, parentId)); } else if (parentId != null && !parsedParentId.equals(Uid.createUid(context.stringBuilder(), parentType, parentId))) {
}
} else if (parentId != null && !parsedParentId.equals(Uid.createUid(context.stringBuilder(), type, parentId))) {
throw new MapperParsingException("Parent id mismatch, document value is [" + Uid.createUid(parsedParentId).id() + "], while external value is [" + parentId + "]"); throw new MapperParsingException("Parent id mismatch, document value is [" + Uid.createUid(parsedParentId).id() + "], while external value is [" + parentId + "]");
} }
} }
@ -295,9 +335,10 @@ public class ParentFieldMapper extends MetadataFieldMapper {
// we have parent mapping, yet no value was set, ignore it... // we have parent mapping, yet no value was set, ignore it...
} }
private SortedDocValuesField createJoinField(String parentType, String id) { private void addJoinFieldIfNeeded(List<Field> fields, MappedFieldType fieldType, String id) {
String joinField = joinField(parentType); if (fieldType.hasDocValues()) {
return new SortedDocValuesField(joinField, new BytesRef(id)); fields.add(new SortedDocValuesField(fieldType.names().indexName(), new BytesRef(id)));
}
} }
public static String joinField(String parentType) { public static String joinField(String parentType) {
@ -309,6 +350,10 @@ public class ParentFieldMapper extends MetadataFieldMapper {
return CONTENT_TYPE; return CONTENT_TYPE;
} }
private boolean joinFieldHasCustomFieldDataSettings() {
return childJoinFieldType != null && childJoinFieldType.fieldDataType() != null && childJoinFieldType.fieldDataType().equals(Defaults.JOIN_FIELD_TYPE.fieldDataType()) == false;
}
@Override @Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
if (!active()) { if (!active()) {
@ -317,9 +362,9 @@ public class ParentFieldMapper extends MetadataFieldMapper {
boolean includeDefaults = params.paramAsBoolean("include_defaults", false); boolean includeDefaults = params.paramAsBoolean("include_defaults", false);
builder.startObject(CONTENT_TYPE); builder.startObject(CONTENT_TYPE);
builder.field("type", type); builder.field("type", parentType);
if (includeDefaults || hasCustomFieldDataSettings()) { if (includeDefaults || joinFieldHasCustomFieldDataSettings()) {
builder.field("fielddata", (Map) fieldType().fieldDataType().getSettings().getAsMap()); builder.field("fielddata", (Map) childJoinFieldType.fieldDataType().getSettings().getAsMap());
} }
builder.endObject(); builder.endObject();
return builder; return builder;
@ -329,8 +374,23 @@ public class ParentFieldMapper extends MetadataFieldMapper {
public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException {
super.merge(mergeWith, mergeResult); super.merge(mergeWith, mergeResult);
ParentFieldMapper fieldMergeWith = (ParentFieldMapper) mergeWith; ParentFieldMapper fieldMergeWith = (ParentFieldMapper) mergeWith;
if (Objects.equals(type, fieldMergeWith.type) == false) { if (Objects.equals(parentType, fieldMergeWith.parentType) == false) {
mergeResult.addConflict("The _parent field's type option can't be changed: [" + type + "]->[" + fieldMergeWith.type + "]"); mergeResult.addConflict("The _parent field's type option can't be changed: [" + parentType + "]->[" + fieldMergeWith.parentType + "]");
}
List<String> conflicts = new ArrayList<>();
fieldType().checkCompatibility(fieldMergeWith.fieldType(), conflicts, true); // always strict, this cannot change
parentJoinFieldType.checkCompatibility(fieldMergeWith.parentJoinFieldType, conflicts, true); // same here
if (childJoinFieldType != null) {
// TODO: this can be set to false when the old parent/child impl is removed, we can do eager global ordinals loading per type.
childJoinFieldType.checkCompatibility(fieldMergeWith.childJoinFieldType, conflicts, mergeResult.updateAllTypes() == false);
}
for (String conflict : conflicts) {
mergeResult.addConflict(conflict);
}
if (active() && mergeResult.simulate() == false && mergeResult.hasConflicts() == false) {
childJoinFieldType = fieldMergeWith.childJoinFieldType.clone();
} }
} }
@ -338,7 +398,7 @@ public class ParentFieldMapper extends MetadataFieldMapper {
* @return Whether the _parent field is actually configured. * @return Whether the _parent field is actually configured.
*/ */
public boolean active() { public boolean active() {
return type != null; return parentType != null;
} }
} }

2
core/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java
View File

@ -245,7 +245,7 @@ public class RootObjectMapper extends ObjectMapper {
if (dynamicTemplate == null) { if (dynamicTemplate == null) {
return null; return null;
} }
Mapper.TypeParser.ParserContext parserContext = context.docMapperParser().parserContext(); Mapper.TypeParser.ParserContext parserContext = context.docMapperParser().parserContext(name);
String mappingType = dynamicTemplate.mappingType(dynamicType); String mappingType = dynamicTemplate.mappingType(dynamicType);
Mapper.TypeParser typeParser = parserContext.typeParser(mappingType); Mapper.TypeParser typeParser = parserContext.typeParser(mappingType);
if (typeParser == null) { if (typeParser == null) {

36
core/src/main/java/org/elasticsearch/search/SearchService.java
View File

@ -62,6 +62,7 @@ import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MappedFieldType.Loading; import org.elasticsearch.index.mapper.MappedFieldType.Loading;
import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
import org.elasticsearch.index.query.TemplateQueryParser; import org.elasticsearch.index.query.TemplateQueryParser;
import org.elasticsearch.index.search.stats.ShardSearchStats; import org.elasticsearch.index.search.stats.ShardSearchStats;
import org.elasticsearch.index.search.stats.StatsGroupsParseElement; import org.elasticsearch.index.search.stats.StatsGroupsParseElement;
@ -910,7 +911,22 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
final Map<String, MappedFieldType> warmUp = new HashMap<>(); final Map<String, MappedFieldType> warmUp = new HashMap<>();
for (DocumentMapper docMapper : mapperService.docMappers(false)) { for (DocumentMapper docMapper : mapperService.docMappers(false)) {
for (FieldMapper fieldMapper : docMapper.mappers()) { for (FieldMapper fieldMapper : docMapper.mappers()) {
final FieldDataType fieldDataType = fieldMapper.fieldType().fieldDataType(); final FieldDataType fieldDataType;
final String indexName;
if (fieldMapper instanceof ParentFieldMapper) {
MappedFieldType joinFieldType = ((ParentFieldMapper) fieldMapper).getChildJoinFieldType();
if (joinFieldType == null) {
continue;
}
fieldDataType = joinFieldType.fieldDataType();
// TODO: this can be removed in 3.0 when the old parent/child impl is removed:
// related to: https://github.com/elastic/elasticsearch/pull/12418
indexName = fieldMapper.fieldType().names().indexName();
} else {
fieldDataType = fieldMapper.fieldType().fieldDataType();
indexName = fieldMapper.fieldType().names().indexName();
}
if (fieldDataType == null) { if (fieldDataType == null) {
continue; continue;
} }
@ -918,7 +934,6 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
continue; continue;
} }
final String indexName = fieldMapper.fieldType().names().indexName();
if (warmUp.containsKey(indexName)) { if (warmUp.containsKey(indexName)) {
continue; continue;
} }
@ -964,14 +979,27 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>(); final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>();
for (DocumentMapper docMapper : mapperService.docMappers(false)) { for (DocumentMapper docMapper : mapperService.docMappers(false)) {
for (FieldMapper fieldMapper : docMapper.mappers()) { for (FieldMapper fieldMapper : docMapper.mappers()) {
final FieldDataType fieldDataType = fieldMapper.fieldType().fieldDataType(); final FieldDataType fieldDataType;
final String indexName;
if (fieldMapper instanceof ParentFieldMapper) {
MappedFieldType joinFieldType = ((ParentFieldMapper) fieldMapper).getChildJoinFieldType();
if (joinFieldType == null) {
continue;
}
fieldDataType = joinFieldType.fieldDataType();
// TODO: this can be removed in 3.0 when the old parent/child impl is removed:
// related to: https://github.com/elastic/elasticsearch/pull/12418
indexName = fieldMapper.fieldType().names().indexName();
} else {
fieldDataType = fieldMapper.fieldType().fieldDataType();
indexName = fieldMapper.fieldType().names().indexName();
}
if (fieldDataType == null) { if (fieldDataType == null) {
continue; continue;
} }
if (fieldDataType.getLoading() != Loading.EAGER_GLOBAL_ORDINALS) { if (fieldDataType.getLoading() != Loading.EAGER_GLOBAL_ORDINALS) {
continue; continue;
} }
final String indexName = fieldMapper.fieldType().names().indexName();
if (warmUpGlobalOrdinals.containsKey(indexName)) { if (warmUpGlobalOrdinals.containsKey(indexName)) {
continue; continue;
} }

2
core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
View File

@ -88,7 +88,7 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase {
} else if (type.getType().equals("geo_point")) { } else if (type.getType().equals("geo_point")) {
fieldType = MapperBuilders.geoPointField(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType(); fieldType = MapperBuilders.geoPointField(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType();
} else if (type.getType().equals("_parent")) { } else if (type.getType().equals("_parent")) {
fieldType = new ParentFieldMapper.Builder().type(fieldName).build(context).fieldType(); fieldType = new ParentFieldMapper.Builder("_type").type(fieldName).build(context).fieldType();
} else if (type.getType().equals("binary")) { } else if (type.getType().equals("binary")) {
fieldType = MapperBuilders.binaryField(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType(); fieldType = MapperBuilders.binaryField(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType();
} else { } else {

158
core/src/test/java/org/elasticsearch/index/mapper/internal/ParentFieldMapperTests.java Normal file
View File

@ -0,0 +1,158 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.internal;
import org.apache.lucene.index.DocValuesType;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.mapper.ContentPath;
import org.elasticsearch.index.mapper.MappedFieldType.Loading;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.test.ESTestCase;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.nullValue;
public class ParentFieldMapperTests extends ESTestCase {
public void testPost2Dot0LazyLoading() {
ParentFieldMapper.Builder builder = new ParentFieldMapper.Builder("child");
builder.type("parent");
builder.fieldDataSettings(createFDSettings(Loading.LAZY));
ParentFieldMapper parentFieldMapper = builder.build(new Mapper.BuilderContext(post2Dot0IndexSettings(), new ContentPath(0)));
assertThat(parentFieldMapper.getParentJoinFieldType().names().indexName(), equalTo("_parent#child"));
assertThat(parentFieldMapper.getParentJoinFieldType().fieldDataType(), nullValue());
assertThat(parentFieldMapper.getParentJoinFieldType().hasDocValues(), is(true));
assertThat(parentFieldMapper.getParentJoinFieldType().docValuesType(), equalTo(DocValuesType.SORTED));
assertThat(parentFieldMapper.getChildJoinFieldType().names().indexName(), equalTo("_parent#parent"));
assertThat(parentFieldMapper.getChildJoinFieldType().fieldDataType().getLoading(), equalTo(Loading.LAZY));
assertThat(parentFieldMapper.getChildJoinFieldType().hasDocValues(), is(true));
assertThat(parentFieldMapper.getChildJoinFieldType().docValuesType(), equalTo(DocValuesType.SORTED));
}
public void testPost2Dot0EagerLoading() {
ParentFieldMapper.Builder builder = new ParentFieldMapper.Builder("child");
builder.type("parent");
builder.fieldDataSettings(createFDSettings(Loading.EAGER));
ParentFieldMapper parentFieldMapper = builder.build(new Mapper.BuilderContext(post2Dot0IndexSettings(), new ContentPath(0)));
assertThat(parentFieldMapper.getParentJoinFieldType().names().indexName(), equalTo("_parent#child"));
assertThat(parentFieldMapper.getParentJoinFieldType().fieldDataType(), nullValue());
assertThat(parentFieldMapper.getParentJoinFieldType().hasDocValues(), is(true));
assertThat(parentFieldMapper.getParentJoinFieldType().docValuesType(), equalTo(DocValuesType.SORTED));
assertThat(parentFieldMapper.getChildJoinFieldType().names().indexName(), equalTo("_parent#parent"));
assertThat(parentFieldMapper.getChildJoinFieldType().fieldDataType().getLoading(), equalTo(Loading.EAGER));
assertThat(parentFieldMapper.getChildJoinFieldType().hasDocValues(), is(true));
assertThat(parentFieldMapper.getChildJoinFieldType().docValuesType(), equalTo(DocValuesType.SORTED));
}
public void testPost2Dot0EagerGlobalOrdinalsLoading() {
ParentFieldMapper.Builder builder = new ParentFieldMapper.Builder("child");
builder.type("parent");
builder.fieldDataSettings(createFDSettings(Loading.EAGER_GLOBAL_ORDINALS));
ParentFieldMapper parentFieldMapper = builder.build(new Mapper.BuilderContext(post2Dot0IndexSettings(), new ContentPath(0)));
assertThat(parentFieldMapper.getParentJoinFieldType().names().indexName(), equalTo("_parent#child"));
assertThat(parentFieldMapper.getParentJoinFieldType().fieldDataType(), nullValue());
assertThat(parentFieldMapper.getParentJoinFieldType().hasDocValues(), is(true));
assertThat(parentFieldMapper.getParentJoinFieldType().docValuesType(), equalTo(DocValuesType.SORTED));
assertThat(parentFieldMapper.getChildJoinFieldType().names().indexName(), equalTo("_parent#parent"));
assertThat(parentFieldMapper.getChildJoinFieldType().fieldDataType().getLoading(), equalTo(Loading.EAGER_GLOBAL_ORDINALS));
assertThat(parentFieldMapper.getChildJoinFieldType().hasDocValues(), is(true));
assertThat(parentFieldMapper.getChildJoinFieldType().docValuesType(), equalTo(DocValuesType.SORTED));
}
public void testPre2Dot0LazyLoading() {
ParentFieldMapper.Builder builder = new ParentFieldMapper.Builder("child");
builder.type("parent");
builder.fieldDataSettings(createFDSettings(Loading.LAZY));
ParentFieldMapper parentFieldMapper = builder.build(new Mapper.BuilderContext(pre2Dot0IndexSettings(), new ContentPath(0)));
assertThat(parentFieldMapper.getParentJoinFieldType().names().indexName(), equalTo("_parent#child"));
assertThat(parentFieldMapper.getParentJoinFieldType().fieldDataType(), nullValue());
assertThat(parentFieldMapper.getParentJoinFieldType().hasDocValues(), is(false));
assertThat(parentFieldMapper.getParentJoinFieldType().docValuesType(), equalTo(DocValuesType.NONE));
assertThat(parentFieldMapper.getChildJoinFieldType().names().indexName(), equalTo("_parent#parent"));
assertThat(parentFieldMapper.getChildJoinFieldType().fieldDataType().getLoading(), equalTo(Loading.LAZY));
assertThat(parentFieldMapper.getChildJoinFieldType().hasDocValues(), is(false));
assertThat(parentFieldMapper.getChildJoinFieldType().docValuesType(), equalTo(DocValuesType.NONE));
}
public void testPre2Dot0EagerLoading() {
ParentFieldMapper.Builder builder = new ParentFieldMapper.Builder("child");
builder.type("parent");
builder.fieldDataSettings(createFDSettings(Loading.EAGER));
ParentFieldMapper parentFieldMapper = builder.build(new Mapper.BuilderContext(pre2Dot0IndexSettings(), new ContentPath(0)));
assertThat(parentFieldMapper.getParentJoinFieldType().names().indexName(), equalTo("_parent#child"));
assertThat(parentFieldMapper.getParentJoinFieldType().fieldDataType(), nullValue());
assertThat(parentFieldMapper.getParentJoinFieldType().hasDocValues(), is(false));
assertThat(parentFieldMapper.getParentJoinFieldType().docValuesType(), equalTo(DocValuesType.NONE));
assertThat(parentFieldMapper.getChildJoinFieldType().names().indexName(), equalTo("_parent#parent"));
assertThat(parentFieldMapper.getChildJoinFieldType().fieldDataType().getLoading(), equalTo(Loading.EAGER));
assertThat(parentFieldMapper.getChildJoinFieldType().hasDocValues(), is(false));
assertThat(parentFieldMapper.getChildJoinFieldType().docValuesType(), equalTo(DocValuesType.NONE));
}
public void testPre2Dot0EagerGlobalOrdinalsLoading() {
ParentFieldMapper.Builder builder = new ParentFieldMapper.Builder("child");
builder.type("parent");
builder.fieldDataSettings(createFDSettings(Loading.EAGER_GLOBAL_ORDINALS));
ParentFieldMapper parentFieldMapper = builder.build(new Mapper.BuilderContext(pre2Dot0IndexSettings(), new ContentPath(0)));
assertThat(parentFieldMapper.getParentJoinFieldType().names().indexName(), equalTo("_parent#child"));
assertThat(parentFieldMapper.getParentJoinFieldType().fieldDataType(), nullValue());
assertThat(parentFieldMapper.getParentJoinFieldType().hasDocValues(), is(false));
assertThat(parentFieldMapper.getParentJoinFieldType().docValuesType(), equalTo(DocValuesType.NONE));
assertThat(parentFieldMapper.getChildJoinFieldType().names().indexName(), equalTo("_parent#parent"));
assertThat(parentFieldMapper.getChildJoinFieldType().fieldDataType().getLoading(), equalTo(Loading.EAGER_GLOBAL_ORDINALS));
assertThat(parentFieldMapper.getChildJoinFieldType().hasDocValues(), is(false));
assertThat(parentFieldMapper.getChildJoinFieldType().docValuesType(), equalTo(DocValuesType.NONE));
}
private static Settings pre2Dot0IndexSettings() {
return Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_6_3).build();
}
private static Settings post2Dot0IndexSettings() {
return Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_1_0).build();
}
private static Settings createFDSettings(Loading loading) {
return new FieldDataType("child", settingsBuilder().put(Loading.KEY, loading)).getSettings();
}
}

2
core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
View File

@ -1199,7 +1199,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
.endObject().endObject()).get(); .endObject().endObject()).get();
fail(); fail();
} catch (MergeMappingException e) { } catch (MergeMappingException e) {
assertThat(e.toString(), containsString("Merge failed with failures {[The _parent field's type option can't be changed: [null]->[parent]]}")); assertThat(e.toString(), containsString("Merge failed with failures {[The _parent field's type option can't be changed: [null]->[parent]"));
} }
} }

15
core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingBwcIT.java
View File

@ -129,14 +129,14 @@ public class ParentFieldLoadingBwcIT extends ESIntegTestCase {
assertAcked(prepareCreate("test") assertAcked(prepareCreate("test")
.setSettings(indexSettings) .setSettings(indexSettings)
.addMapping("parent") .addMapping("parent")
.addMapping("child", childMapping(MappedFieldType.Loading.LAZY)) .addMapping("child", childMapping(MappedFieldType.Loading.LAZY)));
.setUpdateAllTypes(true));
ensureGreen(); ensureGreen();
client().prepareIndex("test", "parent", "1").setSource("{}").get(); client().prepareIndex("test", "parent", "1").setSource("{}").get();
client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get(); client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get();
refresh(); refresh();
IndicesStatsResponse r = client().admin().indices().prepareStats("test").setFieldData(true).setFieldDataFields("*").get();
ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get();
assertThat(response.getIndicesStats().getFieldData().getMemorySizeInBytes(), equalTo(0l)); assertThat(response.getIndicesStats().getFieldData().getMemorySizeInBytes(), equalTo(0l));
@ -145,8 +145,7 @@ public class ParentFieldLoadingBwcIT extends ESIntegTestCase {
assertAcked(prepareCreate("test") assertAcked(prepareCreate("test")
.setSettings(indexSettings) .setSettings(indexSettings)
.addMapping("parent") .addMapping("parent")
.addMapping("child", "_parent", "type=parent") .addMapping("child", "_parent", "type=parent"));
.setUpdateAllTypes(true));
ensureGreen(); ensureGreen();
client().prepareIndex("test", "parent", "1").setSource("{}").get(); client().prepareIndex("test", "parent", "1").setSource("{}").get();
@ -162,8 +161,7 @@ public class ParentFieldLoadingBwcIT extends ESIntegTestCase {
assertAcked(prepareCreate("test") assertAcked(prepareCreate("test")
.setSettings(indexSettings) .setSettings(indexSettings)
.addMapping("parent") .addMapping("parent")
.addMapping("child", childMapping(MappedFieldType.Loading.EAGER)) .addMapping("child", childMapping(MappedFieldType.Loading.EAGER)));
.setUpdateAllTypes(true));
ensureGreen(); ensureGreen();
client().prepareIndex("test", "parent", "1").setSource("{}").get(); client().prepareIndex("test", "parent", "1").setSource("{}").get();
@ -178,8 +176,7 @@ public class ParentFieldLoadingBwcIT extends ESIntegTestCase {
assertAcked(prepareCreate("test") assertAcked(prepareCreate("test")
.setSettings(indexSettings) .setSettings(indexSettings)
.addMapping("parent") .addMapping("parent")
.addMapping("child", childMapping(MappedFieldType.Loading.EAGER_GLOBAL_ORDINALS)) .addMapping("child", childMapping(MappedFieldType.Loading.EAGER_GLOBAL_ORDINALS)));
.setUpdateAllTypes(true));
ensureGreen(); ensureGreen();
// Need to do 2 separate refreshes, otherwise we have 1 segment and then we can't measure if global ordinals // Need to do 2 separate refreshes, otherwise we have 1 segment and then we can't measure if global ordinals
@ -227,7 +224,7 @@ public class ParentFieldLoadingBwcIT extends ESIntegTestCase {
MapperService mapperService = indexService.mapperService(); MapperService mapperService = indexService.mapperService();
DocumentMapper documentMapper = mapperService.documentMapper("child"); DocumentMapper documentMapper = mapperService.documentMapper("child");
if (documentMapper != null) { if (documentMapper != null) {
verified = documentMapper.parentFieldMapper().fieldType().fieldDataType().getLoading() == MappedFieldType.Loading.EAGER_GLOBAL_ORDINALS; verified = documentMapper.parentFieldMapper().getChildJoinFieldType().fieldDataType().getLoading() == MappedFieldType.Loading.EAGER_GLOBAL_ORDINALS;
} }
} }
assertTrue(verified); assertTrue(verified);

14
core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java
View File

@ -57,8 +57,7 @@ public class ParentFieldLoadingIT extends ESIntegTestCase {
assertAcked(prepareCreate("test") assertAcked(prepareCreate("test")
.setSettings(indexSettings) .setSettings(indexSettings)
.addMapping("parent") .addMapping("parent")
.addMapping("child", childMapping(MappedFieldType.Loading.LAZY)) .addMapping("child", childMapping(MappedFieldType.Loading.LAZY)));
.setUpdateAllTypes(true));
ensureGreen(); ensureGreen();
client().prepareIndex("test", "parent", "1").setSource("{}").get(); client().prepareIndex("test", "parent", "1").setSource("{}").get();
@ -73,8 +72,7 @@ public class ParentFieldLoadingIT extends ESIntegTestCase {
assertAcked(prepareCreate("test") assertAcked(prepareCreate("test")
.setSettings(indexSettings) .setSettings(indexSettings)
.addMapping("parent") .addMapping("parent")
.addMapping("child", "_parent", "type=parent") .addMapping("child", "_parent", "type=parent"));
.setUpdateAllTypes(true));
ensureGreen(); ensureGreen();
client().prepareIndex("test", "parent", "1").setSource("{}").get(); client().prepareIndex("test", "parent", "1").setSource("{}").get();
@ -89,8 +87,7 @@ public class ParentFieldLoadingIT extends ESIntegTestCase {
assertAcked(prepareCreate("test") assertAcked(prepareCreate("test")
.setSettings(indexSettings) .setSettings(indexSettings)
.addMapping("parent") .addMapping("parent")
.addMapping("child", childMapping(MappedFieldType.Loading.EAGER)) .addMapping("child", childMapping(MappedFieldType.Loading.EAGER)));
.setUpdateAllTypes(true));
ensureGreen(); ensureGreen();
client().prepareIndex("test", "parent", "1").setSource("{}").get(); client().prepareIndex("test", "parent", "1").setSource("{}").get();
@ -105,8 +102,7 @@ public class ParentFieldLoadingIT extends ESIntegTestCase {
assertAcked(prepareCreate("test") assertAcked(prepareCreate("test")
.setSettings(indexSettings) .setSettings(indexSettings)
.addMapping("parent") .addMapping("parent")
.addMapping("child", childMapping(MappedFieldType.Loading.EAGER_GLOBAL_ORDINALS)) .addMapping("child", childMapping(MappedFieldType.Loading.EAGER_GLOBAL_ORDINALS)));
.setUpdateAllTypes(true));
ensureGreen(); ensureGreen();
// Need to do 2 separate refreshes, otherwise we have 1 segment and then we can't measure if global ordinals // Need to do 2 separate refreshes, otherwise we have 1 segment and then we can't measure if global ordinals
@ -153,7 +149,7 @@ public class ParentFieldLoadingIT extends ESIntegTestCase {
MapperService mapperService = indexService.mapperService(); MapperService mapperService = indexService.mapperService();
DocumentMapper documentMapper = mapperService.documentMapper("child"); DocumentMapper documentMapper = mapperService.documentMapper("child");
if (documentMapper != null) { if (documentMapper != null) {
verified = documentMapper.parentFieldMapper().fieldType().fieldDataType().getLoading() == MappedFieldType.Loading.EAGER_GLOBAL_ORDINALS; verified = documentMapper.parentFieldMapper().getChildJoinFieldType().fieldDataType().getLoading() == MappedFieldType.Loading.EAGER_GLOBAL_ORDINALS;
} }
} }
assertTrue(verified); assertTrue(verified);