honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10593 from jpountz/fix/dynamic_mappings_updates 

Mappings: Same code path for dynamic mappings updates and updates coming from the API.

Close #10593
This commit is contained in:
Adrien Grand 2015-04-16 10:17:38 +02:00
parent 4f8ea78356 563e704881
commit 5806e85771
38 changed files with 804 additions and 535 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dev-tools/create-bwc-index.py
View File

@ -226,6 +226,11 @@ def generate_index(client, version, index_name):
} }
} }
} }
mappings['auto_boost'] = {
'_all': {
'auto_boost': True
}
}
client.indices.create(index=index_name, body={ client.indices.create(index=index_name, body={
'settings': { 'settings': {

21
src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java
View File

@ -19,11 +19,14 @@
package org.elasticsearch.common.lucene.all; package org.elasticsearch.common.lucene.all;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term; import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms; import org.apache.lucene.index.Terms;
import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.payloads.AveragePayloadFunction; import org.apache.lucene.search.payloads.AveragePayloadFunction;
import org.apache.lucene.search.payloads.PayloadTermQuery; import org.apache.lucene.search.payloads.PayloadTermQuery;
import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.search.similarities.Similarity;
@ -119,4 +122,22 @@ public final class AllTermQuery extends PayloadTermQuery {
return true; return true;
} }
@Override
public Query rewrite(IndexReader reader) throws IOException {
boolean hasPayloads = false;
for (LeafReaderContext context : reader.leaves()) {
final Terms terms = context.reader().terms(term.field());
if (terms.hasPayloads()) {
hasPayloads = true;
break;
}
}
if (hasPayloads == false) {
TermQuery rewritten = new TermQuery(term);
rewritten.setBoost(getBoost());
return rewritten;
}
return this;
}
} }

77
src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.index.mapper;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableField;
@ -33,6 +34,7 @@ import org.elasticsearch.ElasticsearchGenerationException;
import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.Preconditions;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.compress.CompressedString;
@ -70,6 +72,7 @@ import org.elasticsearch.script.ScriptService.ScriptType;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
@ -438,10 +441,11 @@ public class DocumentMapper implements ToXContent {
ParseContext.InternalParseContext context = cache.get(); ParseContext.InternalParseContext context = cache.get();
if (source.type() != null && !source.type().equals(this.type)) { if (source.type() != null && !source.type().equals(this.type)) {
throw new MapperParsingException("Type mismatch, provide type [" + source.type() + "] but mapper is of type [" + this.type + "]", context.mappingsModified()); throw new MapperParsingException("Type mismatch, provide type [" + source.type() + "] but mapper is of type [" + this.type + "]");
} }
source.type(this.type); source.type(this.type);
boolean mappingsModified = false;
XContentParser parser = source.parser(); XContentParser parser = source.parser();
try { try {
if (parser == null) { if (parser == null) {
@ -456,7 +460,7 @@ public class DocumentMapper implements ToXContent {
int countDownTokens = 0; int countDownTokens = 0;
XContentParser.Token token = parser.nextToken(); XContentParser.Token token = parser.nextToken();
if (token != XContentParser.Token.START_OBJECT) { if (token != XContentParser.Token.START_OBJECT) {
throw new MapperParsingException("Malformed content, must start with an object", context.mappingsModified()); throw new MapperParsingException("Malformed content, must start with an object");
} }
boolean emptyDoc = false; boolean emptyDoc = false;
token = parser.nextToken(); token = parser.nextToken();
@ -464,7 +468,7 @@ public class DocumentMapper implements ToXContent {
// empty doc, we can handle it... // empty doc, we can handle it...
emptyDoc = true; emptyDoc = true;
} else if (token != XContentParser.Token.FIELD_NAME) { } else if (token != XContentParser.Token.FIELD_NAME) {
throw new MapperParsingException("Malformed content, after first object, either the type field or the actual properties should exist", context.mappingsModified()); throw new MapperParsingException("Malformed content, after first object, either the type field or the actual properties should exist");
} }
for (RootMapper rootMapper : rootMappersOrdered) { for (RootMapper rootMapper : rootMappersOrdered) {
@ -472,7 +476,31 @@ public class DocumentMapper implements ToXContent {
} }
if (!emptyDoc) { if (!emptyDoc) {
rootObjectMapper.parse(context); Mapper update = rootObjectMapper.parse(context);
for (RootObjectMapper mapper : context.updates()) {
if (update == null) {
update = mapper;
} else {
MapperUtils.merge(update, mapper);
}
}
if (update != null) {
// TODO: validate the mapping update on the master node
// lock to avoid concurrency issues with mapping updates coming from the API
synchronized(this) {
// simulate on the first time to check if the mapping update is applicable
MergeContext mergeContext = newMmergeContext(new MergeFlags().simulate(true));
rootObjectMapper.merge(update, mergeContext);
if (mergeContext.hasConflicts()) {
throw new MapperParsingException("Could not apply generated dynamic mappings: " + Arrays.toString(mergeContext.buildConflicts()));
} else {
// then apply it for real
mappingsModified = true;
mergeContext = newMmergeContext(new MergeFlags().simulate(false));
rootObjectMapper.merge(update, mergeContext);
}
}
}
} }
for (int i = 0; i < countDownTokens; i++) { for (int i = 0; i < countDownTokens; i++) {
@ -490,10 +518,10 @@ public class DocumentMapper implements ToXContent {
// Throw a more meaningful message if the document is empty. // Throw a more meaningful message if the document is empty.
if (source.source() != null && source.source().length() == 0) { if (source.source() != null && source.source().length() == 0) {
throw new MapperParsingException("failed to parse, document is empty", context.mappingsModified()); throw new MapperParsingException("failed to parse, document is empty");
} }
throw new MapperParsingException("failed to parse", e, context.mappingsModified()); throw new MapperParsingException("failed to parse", e);
} finally { } finally {
// only close the parser when its not provided externally // only close the parser when its not provided externally
if (source.parser() == null && parser != null) { if (source.parser() == null && parser != null) {
@ -521,7 +549,7 @@ public class DocumentMapper implements ToXContent {
} }
ParsedDocument doc = new ParsedDocument(context.uid(), context.version(), context.id(), context.type(), source.routing(), source.timestamp(), source.ttl(), context.docs(), ParsedDocument doc = new ParsedDocument(context.uid(), context.version(), context.id(), context.type(), source.routing(), source.timestamp(), source.ttl(), context.docs(),
context.source(), context.mappingsModified()).parent(source.parent()); context.source(), mappingsModified).parent(source.parent());
// reset the context to free up memory // reset the context to free up memory
context.reset(null, null, null, null); context.reset(null, null, null, null);
return doc; return doc;
@ -637,8 +665,41 @@ public class DocumentMapper implements ToXContent {
rootObjectMapper.traverse(listener); rootObjectMapper.traverse(listener);
} }
private MergeContext newMmergeContext(MergeFlags mergeFlags) {
return new MergeContext(mergeFlags) {
List<String> conflicts = new ArrayList<>();
@Override
public void addFieldMappers(List<FieldMapper<?>> fieldMappers) {
DocumentMapper.this.addFieldMappers(fieldMappers);
}
@Override
public void addObjectMappers(Collection<ObjectMapper> objectMappers) {
DocumentMapper.this.addObjectMappers(objectMappers);
}
@Override
public void addConflict(String mergeFailure) {
conflicts.add(mergeFailure);
}
@Override
public boolean hasConflicts() {
return conflicts.isEmpty() == false;
}
@Override
public String[] buildConflicts() {
return conflicts.toArray(Strings.EMPTY_ARRAY);
}
};
}
public synchronized MergeResult merge(DocumentMapper mergeWith, MergeFlags mergeFlags) { public synchronized MergeResult merge(DocumentMapper mergeWith, MergeFlags mergeFlags) {
MergeContext mergeContext = new MergeContext(this, mergeFlags); final MergeContext mergeContext = newMmergeContext(mergeFlags);
assert rootMappers.size() == mergeWith.rootMappers.size(); assert rootMappers.size() == mergeWith.rootMappers.size();
rootObjectMapper.merge(mergeWith.rootObjectMapper, mergeContext); rootObjectMapper.merge(mergeWith.rootObjectMapper, mergeContext);

7
src/main/java/org/elasticsearch/index/mapper/Mapper.java
View File

@ -125,7 +125,12 @@ public interface Mapper extends ToXContent {
String name(); String name();
void parse(ParseContext context) throws IOException; /**
* Parse using the provided {@link ParseContext} and return a mapping
* update if dynamic mappings modified the mappings, or {@code null} if
* mappings were not modified.
*/
Mapper parse(ParseContext context) throws IOException;
void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappingException; void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappingException;

18
src/main/java/org/elasticsearch/index/mapper/MapperParsingException.java
View File

@ -28,28 +28,10 @@ public class MapperParsingException extends MapperException {
public MapperParsingException(String message) { public MapperParsingException(String message) {
super(message); super(message);
mappingsModified = false;
}
public boolean isMappingsModified() {
return mappingsModified;
}
private boolean mappingsModified = false;
public MapperParsingException(String message, boolean mappingsModified) {
super(message);
this.mappingsModified = mappingsModified;
}
public MapperParsingException(String message, Throwable cause, boolean mappingsModified) {
super(message, cause);
this.mappingsModified = mappingsModified;
} }
public MapperParsingException(String message, Throwable cause) { public MapperParsingException(String message, Throwable cause) {
super(message, cause); super(message, cause);
this.mappingsModified = false;
} }
@Override @Override

82
src/main/java/org/elasticsearch/index/mapper/MapperUtils.java Normal file
View File

@ -0,0 +1,82 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.index.mapper.object.ObjectMapper;
import org.elasticsearch.index.mapper.object.RootObjectMapper;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
public enum MapperUtils {
;
/**
* Parse the given {@code context} with the given {@code mapper} and apply
* the potential mapping update in-place. This method is useful when
* composing mapping updates.
*/
public static <M extends Mapper> M parseAndMergeUpdate(M mapper, ParseContext context) throws IOException {
final Mapper update = mapper.parse(context);
if (update != null) {
merge(mapper, update);
}
return mapper;
}
/**
* Merge {@code mergeWith} into {@code mergeTo}. Note: this method only
* merges mappings, not lookup structures. Conflicts are returned as exceptions.
*/
public static void merge(Mapper mergeInto, Mapper mergeWith) {
MergeContext ctx = new MergeContext(new DocumentMapper.MergeFlags().simulate(false)) {
@Override
public boolean hasConflicts() {
return false;
}
@Override
public String[] buildConflicts() {
return Strings.EMPTY_ARRAY;
}
@Override
public void addObjectMappers(Collection<ObjectMapper> objectMappers) {
// no-op
}
@Override
public void addFieldMappers(List<FieldMapper<?>> fieldMappers) {
// no-op
}
@Override
public void addConflict(String mergeFailure) {
throw new ElasticsearchIllegalStateException("Merging dynamic updates triggered a conflict: " + mergeFailure);
}
};
mergeInto.merge(mergeWith, ctx);
}
}

28
src/main/java/org/elasticsearch/index/mapper/MergeContext.java
View File

@ -19,41 +19,33 @@
package org.elasticsearch.index.mapper; package org.elasticsearch.index.mapper;
import com.google.common.collect.Lists; import org.elasticsearch.index.mapper.object.ObjectMapper;
import java.util.Collection;
import java.util.List; import java.util.List;
/** /**
* *
*/ */
public class MergeContext { public abstract class MergeContext {
private final DocumentMapper documentMapper;
private final DocumentMapper.MergeFlags mergeFlags; private final DocumentMapper.MergeFlags mergeFlags;
private final List<String> mergeConflicts = Lists.newArrayList();
public MergeContext(DocumentMapper documentMapper, DocumentMapper.MergeFlags mergeFlags) { public MergeContext(DocumentMapper.MergeFlags mergeFlags) {
this.documentMapper = documentMapper;
this.mergeFlags = mergeFlags; this.mergeFlags = mergeFlags;
} }
public DocumentMapper docMapper() { public abstract void addFieldMappers(List<FieldMapper<?>> fieldMappers);
return documentMapper;
} public abstract void addObjectMappers(Collection<ObjectMapper> objectMappers);
public DocumentMapper.MergeFlags mergeFlags() { public DocumentMapper.MergeFlags mergeFlags() {
return mergeFlags; return mergeFlags;
} }
public void addConflict(String mergeFailure) { public abstract void addConflict(String mergeFailure);
mergeConflicts.add(mergeFailure);
}
public boolean hasConflicts() { public abstract boolean hasConflicts();
return !mergeConflicts.isEmpty();
}
public String[] buildConflicts() { public abstract String[] buildConflicts();
return mergeConflicts.toArray(new String[mergeConflicts.size()]);
}
} }

103
src/main/java/org/elasticsearch/index/mapper/ParseContext.java
View File

@ -22,7 +22,7 @@ package org.elasticsearch.index.mapper;
import com.carrotsearch.hppc.ObjectObjectMap; import com.carrotsearch.hppc.ObjectObjectMap;
import com.carrotsearch.hppc.ObjectObjectOpenHashMap; import com.carrotsearch.hppc.ObjectObjectOpenHashMap;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableField;
@ -38,7 +38,11 @@ import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.mapper.DocumentMapper.ParseListener; import org.elasticsearch.index.mapper.DocumentMapper.ParseListener;
import org.elasticsearch.index.mapper.object.RootObjectMapper; import org.elasticsearch.index.mapper.object.RootObjectMapper;
import java.util.*; import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
/** /**
* *
@ -194,31 +198,6 @@ public abstract class ParseContext {
return in.docMapperParser(); return in.docMapperParser();
} }
@Override
public boolean mappingsModified() {
return in.mappingsModified();
}
@Override
public void setMappingsModified() {
in.setMappingsModified();
}
@Override
public void setWithinNewMapper() {
in.setWithinNewMapper();
}
@Override
public void clearWithinNewMapper() {
in.clearWithinNewMapper();
}
@Override
public boolean isWithinNewMapper() {
return in.isWithinNewMapper();
}
@Override @Override
public boolean isWithinCopyTo() { public boolean isWithinCopyTo() {
return in.isWithinCopyTo(); return in.isWithinCopyTo();
@ -379,6 +358,15 @@ public abstract class ParseContext {
return in.stringBuilder(); return in.stringBuilder();
} }
@Override
public void addRootObjectUpdate(RootObjectMapper update) {
in.addRootObjectUpdate(update);
}
@Override
public List<RootObjectMapper> updates() {
return in.updates();
}
} }
public static class InternalParseContext extends ParseContext { public static class InternalParseContext extends ParseContext {
@ -414,12 +402,13 @@ public abstract class ParseContext {
private Map<String, String> ignoredValues = new HashMap<>(); private Map<String, String> ignoredValues = new HashMap<>();
private boolean mappingsModified = false; private boolean mappingsModified = false;
private boolean withinNewMapper = false;
private AllEntries allEntries = new AllEntries(); private AllEntries allEntries = new AllEntries();
private float docBoost = 1.0f; private float docBoost = 1.0f;
private final List<RootObjectMapper> rootMapperDynamicUpdates = new ArrayList<>();
public InternalParseContext(String index, @Nullable Settings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper, ContentPath path) { public InternalParseContext(String index, @Nullable Settings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper, ContentPath path) {
this.index = index; this.index = index;
this.indexSettings = indexSettings; this.indexSettings = indexSettings;
@ -444,11 +433,11 @@ public abstract class ParseContext {
this.source = source == null ? null : sourceToParse.source(); this.source = source == null ? null : sourceToParse.source();
this.path.reset(); this.path.reset();
this.mappingsModified = false; this.mappingsModified = false;
this.withinNewMapper = false;
this.listener = listener == null ? DocumentMapper.ParseListener.EMPTY : listener; this.listener = listener == null ? DocumentMapper.ParseListener.EMPTY : listener;
this.allEntries = new AllEntries(); this.allEntries = new AllEntries();
this.ignoredValues.clear(); this.ignoredValues.clear();
this.docBoost = 1.0f; this.docBoost = 1.0f;
this.rootMapperDynamicUpdates.clear();
} }
@Override @Override
@ -461,31 +450,6 @@ public abstract class ParseContext {
return this.docMapperParser; return this.docMapperParser;
} }
@Override
public boolean mappingsModified() {
return this.mappingsModified;
}
@Override
public void setMappingsModified() {
this.mappingsModified = true;
}
@Override
public void setWithinNewMapper() {
this.withinNewMapper = true;
}
@Override
public void clearWithinNewMapper() {
this.withinNewMapper = false;
}
@Override
public boolean isWithinNewMapper() {
return withinNewMapper;
}
@Override @Override
public String index() { public String index() {
return this.index; return this.index;
@ -638,22 +602,22 @@ public abstract class ParseContext {
stringBuilder.setLength(0); stringBuilder.setLength(0);
return this.stringBuilder; return this.stringBuilder;
} }
@Override
public void addRootObjectUpdate(RootObjectMapper mapper) {
rootMapperDynamicUpdates.add(mapper);
}
@Override
public List<RootObjectMapper> updates() {
return rootMapperDynamicUpdates;
}
} }
public abstract boolean flyweight(); public abstract boolean flyweight();
public abstract DocumentMapperParser docMapperParser(); public abstract DocumentMapperParser docMapperParser();
public abstract boolean mappingsModified();
public abstract void setMappingsModified();
public abstract void setWithinNewMapper();
public abstract void clearWithinNewMapper();
public abstract boolean isWithinNewMapper();
/** /**
* Return a new context that will be within a copy-to operation. * Return a new context that will be within a copy-to operation.
*/ */
@ -854,4 +818,15 @@ public abstract class ParseContext {
*/ */
public abstract StringBuilder stringBuilder(); public abstract StringBuilder stringBuilder();
/**
* Add a dynamic update to the root object mapper.
* TODO: can we nuke it, it is only needed for copy_to
*/
public abstract void addRootObjectUpdate(RootObjectMapper update);
/**
* Get dynamic updates to the root object mapper.
* TODO: can we nuke it, it is only needed for copy_to
*/
public abstract List<RootObjectMapper> updates();
} }

4
src/main/java/org/elasticsearch/index/mapper/StrictDynamicMappingException.java
View File

@ -24,8 +24,8 @@ import org.elasticsearch.rest.RestStatus;
*/ */
public class StrictDynamicMappingException extends MapperParsingException { public class StrictDynamicMappingException extends MapperParsingException {
public StrictDynamicMappingException(String path, String fieldName, boolean mappingsModified) { public StrictDynamicMappingException(String path, String fieldName) {
super("mapping set to strict, dynamic introduction of [" + fieldName + "] within [" + path + "] is not allowed", mappingsModified); super("mapping set to strict, dynamic introduction of [" + fieldName + "] within [" + path + "] is not allowed");
} }
@Override @Override

81
src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java
View File

@ -44,6 +44,7 @@ import org.apache.lucene.search.TermRangeFilter;
import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.Version; import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.ImmutableOpenMap;
@ -70,6 +71,7 @@ import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.mapper.ParseContext.Document;
import org.elasticsearch.index.mapper.internal.AllFieldMapper; import org.elasticsearch.index.mapper.internal.AllFieldMapper;
import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.mapper.object.ObjectMapper;
import org.elasticsearch.index.mapper.object.RootObjectMapper;
import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.search.FieldDataTermsFilter; import org.elasticsearch.index.search.FieldDataTermsFilter;
import org.elasticsearch.index.similarity.SimilarityLookupService; import org.elasticsearch.index.similarity.SimilarityLookupService;
@ -81,7 +83,6 @@ import java.util.Arrays;
import java.util.Comparator; import java.util.Comparator;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map;
import java.util.TreeMap; import java.util.TreeMap;
/** /**
@ -434,7 +435,7 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T> {
} }
@Override @Override
public void parse(ParseContext context) throws IOException { public Mapper parse(ParseContext context) throws IOException {
final List<Field> fields = new ArrayList<>(2); final List<Field> fields = new ArrayList<>(2);
try { try {
parseCreateField(context, fields); parseCreateField(context, fields);
@ -447,12 +448,13 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T> {
} }
} }
} catch (Exception e) { } catch (Exception e) {
throw new MapperParsingException("failed to parse [" + names.fullName() + "]", e, context.mappingsModified()); throw new MapperParsingException("failed to parse [" + names.fullName() + "]", e);
} }
multiFields.parse(this, context); multiFields.parse(this, context);
if (copyTo != null) { if (copyTo != null) {
copyTo.parse(context); copyTo.parse(context);
} }
return null;
} }
/** /**
@ -968,7 +970,7 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T> {
// first add all field mappers // first add all field mappers
if (newFieldMappers != null) { if (newFieldMappers != null) {
mergeContext.docMapper().addFieldMappers(newFieldMappers); mergeContext.addFieldMappers(newFieldMappers);
} }
// now publish mappers // now publish mappers
if (newMappersBuilder != null) { if (newMappersBuilder != null) {
@ -1089,54 +1091,41 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T> {
// The path of the dest field might be completely different from the current one so we need to reset it // The path of the dest field might be completely different from the current one so we need to reset it
context = context.overridePath(new ContentPath(0)); context = context.overridePath(new ContentPath(0));
ObjectMapper mapper = context.root();
String objectPath = "";
String fieldPath = field;
int posDot = field.lastIndexOf('.'); int posDot = field.lastIndexOf('.');
if (posDot > 0) { if (posDot > 0) {
// Compound name objectPath = field.substring(0, posDot);
String objectPath = field.substring(0, posDot);
String fieldPath = field.substring(posDot + 1);
ObjectMapper mapper = context.docMapper().objectMappers().get(objectPath);
if (mapper == null) {
//TODO: Create an object dynamically?
throw new MapperParsingException("attempt to copy value to non-existing object [" + field + "]", context.mappingsModified());
}
context.path().add(objectPath); context.path().add(objectPath);
mapper = context.docMapper().objectMappers().get(objectPath);
// We might be in dynamically created field already, so need to clean withinNewMapper flag fieldPath = field.substring(posDot + 1);
// and then restore it, so we wouldn't miss new mappers created from copy_to fields
boolean origWithinNewMapper = context.isWithinNewMapper();
context.clearWithinNewMapper();
try {
mapper.parseDynamicValue(context, fieldPath, context.parser().currentToken());
} finally {
if (origWithinNewMapper) {
context.setWithinNewMapper();
} else {
context.clearWithinNewMapper();
}
}
} else {
// We might be in dynamically created field already, so need to clean withinNewMapper flag
// and then restore it, so we wouldn't miss new mappers created from copy_to fields
boolean origWithinNewMapper = context.isWithinNewMapper();
context.clearWithinNewMapper();
try {
context.docMapper().root().parseDynamicValue(context, field, context.parser().currentToken());
} finally {
if (origWithinNewMapper) {
context.setWithinNewMapper();
} else {
context.clearWithinNewMapper();
}
}
} }
if (mapper == null) {
//TODO: Create an object dynamically?
throw new MapperParsingException("attempt to copy value to non-existing object [" + field + "]");
}
ObjectMapper update = mapper.parseDynamicValue(context, fieldPath, context.parser().currentToken());
assert update != null; // we are parsing a dynamic value so we necessarily created a new mapping
// propagate the update to the root
while (objectPath.length() > 0) {
String parentPath = "";
ObjectMapper parent = context.root();
posDot = objectPath.lastIndexOf('.');
if (posDot > 0) {
parentPath = objectPath.substring(0, posDot);
parent = context.docMapper().objectMappers().get(parentPath);
}
if (parent == null) {
throw new ElasticsearchIllegalStateException("[" + objectPath + "] has no parent for path [" + parentPath + "]");
}
update = parent.mappingUpdate(update);
objectPath = parentPath;
}
context.addRootObjectUpdate((RootObjectMapper) update);
} }
} }
} }
/** /**

3
src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java
View File

@ -266,7 +266,7 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
} }
@Override @Override
public void parse(ParseContext context) throws IOException { public Mapper parse(ParseContext context) throws IOException {
XContentParser parser = context.parser(); XContentParser parser = context.parser();
XContentParser.Token token = parser.currentToken(); XContentParser.Token token = parser.currentToken();
@ -382,6 +382,7 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
context.doc().add(getCompletionField(ctx, input, suggestPayload)); context.doc().add(getCompletionField(ctx, input, suggestPayload));
} }
} }
return null;
} }
private void checkWeight(long weight) { private void checkWeight(long weight) {

3
src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
View File

@ -515,7 +515,7 @@ public class GeoPointFieldMapper extends AbstractFieldMapper<GeoPoint> implement
} }
@Override @Override
public void parse(ParseContext context) throws IOException { public Mapper parse(ParseContext context) throws IOException {
ContentPath.Type origPathType = context.path().pathType(); ContentPath.Type origPathType = context.path().pathType();
context.path().pathType(pathType); context.path().pathType(pathType);
context.path().add(name()); context.path().add(name());
@ -565,6 +565,7 @@ public class GeoPointFieldMapper extends AbstractFieldMapper<GeoPoint> implement
context.path().remove(); context.path().remove();
context.path().pathType(origPathType); context.path().pathType(origPathType);
return null;
} }
private void parseGeohashField(ParseContext context, String geohash) throws IOException { private void parseGeohashField(ParseContext context, String geohash) throws IOException {

7
src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java
View File

@ -237,19 +237,19 @@ public class GeoShapeFieldMapper extends AbstractFieldMapper<String> {
} }
@Override @Override
public void parse(ParseContext context) throws IOException { public Mapper parse(ParseContext context) throws IOException {
try { try {
Shape shape = context.parseExternalValue(Shape.class); Shape shape = context.parseExternalValue(Shape.class);
if (shape == null) { if (shape == null) {
ShapeBuilder shapeBuilder = ShapeBuilder.parse(context.parser(), this); ShapeBuilder shapeBuilder = ShapeBuilder.parse(context.parser(), this);
if (shapeBuilder == null) { if (shapeBuilder == null) {
return; return null;
} }
shape = shapeBuilder.build(); shape = shapeBuilder.build();
} }
Field[] fields = defaultStrategy.createIndexableFields(shape); Field[] fields = defaultStrategy.createIndexableFields(shape);
if (fields == null || fields.length == 0) { if (fields == null || fields.length == 0) {
return; return null;
} }
for (Field field : fields) { for (Field field : fields) {
if (!customBoost()) { if (!customBoost()) {
@ -262,6 +262,7 @@ public class GeoShapeFieldMapper extends AbstractFieldMapper<String> {
} catch (Exception e) { } catch (Exception e) {
throw new MapperParsingException("failed to parse [" + names.fullName() + "]", e); throw new MapperParsingException("failed to parse [" + names.fullName() + "]", e);
} }
return null;
} }
@Override @Override

45
src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
View File

@ -25,7 +25,6 @@ import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.Term; import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query; import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.elasticsearch.Version; import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings; import org.elasticsearch.common.Strings;
@ -33,9 +32,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.all.AllField; import org.elasticsearch.common.lucene.all.AllField;
import org.elasticsearch.common.lucene.all.AllTermQuery; import org.elasticsearch.common.lucene.all.AllTermQuery;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.loader.SettingsLoader;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.fielddata.FieldDataType;
@ -97,9 +94,6 @@ public class AllFieldMapper extends AbstractFieldMapper<String> implements Inter
private EnabledAttributeMapper enabled = Defaults.ENABLED; private EnabledAttributeMapper enabled = Defaults.ENABLED;
// an internal flag, automatically set if we encounter boosting
boolean autoBoost = false;
public Builder() { public Builder() {
super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE)); super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE));
builder = this; builder = this;
@ -120,7 +114,7 @@ public class AllFieldMapper extends AbstractFieldMapper<String> implements Inter
} }
fieldType.setTokenized(true); fieldType.setTokenized(true);
return new AllFieldMapper(name, fieldType, indexAnalyzer, searchAnalyzer, enabled, autoBoost, similarity, normsLoading, fieldDataSettings, context.indexSettings()); return new AllFieldMapper(name, fieldType, indexAnalyzer, searchAnalyzer, enabled, similarity, normsLoading, fieldDataSettings, context.indexSettings());
} }
} }
@ -154,8 +148,8 @@ public class AllFieldMapper extends AbstractFieldMapper<String> implements Inter
if (fieldName.equals("enabled")) { if (fieldName.equals("enabled")) {
builder.enabled(nodeBooleanValue(fieldNode) ? EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED); builder.enabled(nodeBooleanValue(fieldNode) ? EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED);
iterator.remove(); iterator.remove();
} else if (fieldName.equals("auto_boost")) { } else if (fieldName.equals("auto_boost") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
builder.autoBoost = nodeBooleanValue(fieldNode); // Old 1.x setting which is now ignored
iterator.remove(); iterator.remove();
} }
} }
@ -165,24 +159,17 @@ public class AllFieldMapper extends AbstractFieldMapper<String> implements Inter
private EnabledAttributeMapper enabledState; private EnabledAttributeMapper enabledState;
// The autoBoost flag is automatically set based on indexed docs on the mappings
// if a doc is indexed with a specific boost value and part of _all, it is automatically
// set to true. This allows to optimize (automatically, which we like) for the common case
// where fields don't usually have boost associated with them, and we don't need to use the
// special SpanTermQuery to look at payloads
private volatile boolean autoBoost;
public AllFieldMapper(Settings indexSettings) { public AllFieldMapper(Settings indexSettings) {
this(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE), null, null, Defaults.ENABLED, false, null, null, null, indexSettings); this(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE), null, null, Defaults.ENABLED, null, null, null, indexSettings);
} }
protected AllFieldMapper(String name, FieldType fieldType, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer, protected AllFieldMapper(String name, FieldType fieldType, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer,
EnabledAttributeMapper enabled, boolean autoBoost, SimilarityProvider similarity, Loading normsLoading, EnabledAttributeMapper enabled, SimilarityProvider similarity, Loading normsLoading,
@Nullable Settings fieldDataSettings, Settings indexSettings) { @Nullable Settings fieldDataSettings, Settings indexSettings) {
super(new Names(name, name, name, name), 1.0f, fieldType, false, indexAnalyzer, searchAnalyzer, super(new Names(name, name, name, name), 1.0f, fieldType, false, indexAnalyzer, searchAnalyzer,
similarity, normsLoading, fieldDataSettings, indexSettings); similarity, normsLoading, fieldDataSettings, indexSettings);
this.enabledState = enabled; this.enabledState = enabled;
this.autoBoost = autoBoost;
} }
@ -202,13 +189,7 @@ public class AllFieldMapper extends AbstractFieldMapper<String> implements Inter
@Override @Override
public Query queryStringTermQuery(Term term) { public Query queryStringTermQuery(Term term) {
if (!autoBoost) { return new AllTermQuery(term);
return new TermQuery(term);
}
if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {
return new AllTermQuery(term);
}
return new TermQuery(term);
} }
@Override @Override
@ -226,8 +207,9 @@ public class AllFieldMapper extends AbstractFieldMapper<String> implements Inter
} }
@Override @Override
public void parse(ParseContext context) throws IOException { public Mapper parse(ParseContext context) throws IOException {
// we parse in post parse // we parse in post parse
return null;
} }
@Override @Override
@ -242,14 +224,6 @@ public class AllFieldMapper extends AbstractFieldMapper<String> implements Inter
} }
// reset the entries // reset the entries
context.allEntries().reset(); context.allEntries().reset();
// if the autoBoost flag is not set, and we indexed a doc with custom boost, make
// sure to update the flag, and notify mappings on change
if (!autoBoost && context.allEntries().customBoost()) {
autoBoost = true;
context.setMappingsModified();
}
Analyzer analyzer = findAnalyzer(context); Analyzer analyzer = findAnalyzer(context);
fields.add(new AllField(names.indexName(), context.allEntries(), analyzer, fieldType)); fields.add(new AllField(names.indexName(), context.allEntries(), analyzer, fieldType));
} }
@ -305,9 +279,6 @@ public class AllFieldMapper extends AbstractFieldMapper<String> implements Inter
if (includeDefaults || enabledState != Defaults.ENABLED) { if (includeDefaults || enabledState != Defaults.ENABLED) {
builder.field("enabled", enabledState.enabled); builder.field("enabled", enabledState.enabled);
} }
if (includeDefaults || autoBoost != false) {
builder.field("auto_boost", autoBoost);
}
if (includeDefaults || fieldType.stored() != Defaults.FIELD_TYPE.stored()) { if (includeDefaults || fieldType.stored() != Defaults.FIELD_TYPE.stored()) {
builder.field("store", fieldType.stored()); builder.field("store", fieldType.stored());
} }

3
src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java
View File

@ -184,8 +184,9 @@ public class FieldNamesFieldMapper extends AbstractFieldMapper<String> implement
} }
@Override @Override
public void parse(ParseContext context) throws IOException { public Mapper parse(ParseContext context) throws IOException {
// we parse in post parse // we parse in post parse
return null;
} }
@Override @Override

9
src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
View File

@ -307,16 +307,11 @@ public class IdFieldMapper extends AbstractFieldMapper<String> implements Intern
@Override @Override
public void postParse(ParseContext context) throws IOException { public void postParse(ParseContext context) throws IOException {
if (context.id() == null && !context.sourceToParse().flyweight()) { if (context.id() == null && !context.sourceToParse().flyweight()) {
throw new MapperParsingException("No id found while parsing the content source", context.mappingsModified()); throw new MapperParsingException("No id found while parsing the content source");
} }
// it either get built in the preParse phase, or get parsed... // it either get built in the preParse phase, or get parsed...
} }
@Override
public void parse(ParseContext context) throws IOException {
super.parse(context);
}
@Override @Override
public boolean includeInObject() { public boolean includeInObject() {
return true; return true;
@ -329,7 +324,7 @@ public class IdFieldMapper extends AbstractFieldMapper<String> implements Intern
// we are in the parse Phase // we are in the parse Phase
String id = parser.text(); String id = parser.text();
if (context.id() != null && !context.id().equals(id)) { if (context.id() != null && !context.id().equals(id)) {
throw new MapperParsingException("Provided id [" + context.id() + "] does not match the content one [" + id + "]", context.mappingsModified()); throw new MapperParsingException("Provided id [" + context.id() + "] does not match the content one [" + id + "]");
} }
context.id(id); context.id(id);
} // else we are in the pre/post parse phase } // else we are in the pre/post parse phase

4
src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
View File

@ -166,8 +166,8 @@ public class IndexFieldMapper extends AbstractFieldMapper<String> implements Int
} }
@Override @Override
public void parse(ParseContext context) throws IOException { public Mapper parse(ParseContext context) throws IOException {
return null;
} }
@Override @Override

3
src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
View File

@ -182,10 +182,11 @@ public class RoutingFieldMapper extends AbstractFieldMapper<String> implements I
} }
@Override @Override
public void parse(ParseContext context) throws IOException { public Mapper parse(ParseContext context) throws IOException {
// no need ot parse here, we either get the routing in the sourceToParse // no need ot parse here, we either get the routing in the sourceToParse
// or we don't have routing, if we get it in sourceToParse, we process it in preParse // or we don't have routing, if we get it in sourceToParse, we process it in preParse
// which will always be called // which will always be called
return null;
} }
@Override @Override

3
src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java
View File

@ -134,8 +134,9 @@ public class SizeFieldMapper extends IntegerFieldMapper implements RootMapper {
} }
@Override @Override
public void parse(ParseContext context) throws IOException { public Mapper parse(ParseContext context) throws IOException {
// nothing to do here, we call the parent in postParse // nothing to do here, we call the parent in postParse
return null;
} }
@Override @Override

3
src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
View File

@ -251,8 +251,9 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements In
} }
@Override @Override
public void parse(ParseContext context) throws IOException { public Mapper parse(ParseContext context) throws IOException {
// nothing to do here, we will call it in pre parse // nothing to do here, we will call it in pre parse
return null;
} }
@Override @Override

3
src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
View File

@ -175,7 +175,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R
} }
@Override @Override
public void parse(ParseContext context) throws IOException, MapperParsingException { public Mapper parse(ParseContext context) throws IOException, MapperParsingException {
if (context.sourceToParse().ttl() < 0) { // no ttl has been provided externally if (context.sourceToParse().ttl() < 0) { // no ttl has been provided externally
long ttl; long ttl;
if (context.parser().currentToken() == XContentParser.Token.VALUE_STRING) { if (context.parser().currentToken() == XContentParser.Token.VALUE_STRING) {
@ -188,6 +188,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R
} }
context.sourceToParse().ttl(ttl); context.sourceToParse().ttl(ttl);
} }
return null;
} }
@Override @Override

3
src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
View File

@ -273,8 +273,9 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
} }
@Override @Override
public void parse(ParseContext context) throws IOException { public Mapper parse(ParseContext context) throws IOException {
// nothing to do here, we call the parent in preParse // nothing to do here, we call the parent in preParse
return null;
} }
@Override @Override

3
src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
View File

@ -158,8 +158,9 @@ public class TypeFieldMapper extends AbstractFieldMapper<String> implements Inte
} }
@Override @Override
public void parse(ParseContext context) throws IOException { public Mapper parse(ParseContext context) throws IOException {
// we parse in pre parse // we parse in pre parse
return null;
} }
@Override @Override

3
src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
View File

@ -167,8 +167,9 @@ public class UidFieldMapper extends AbstractFieldMapper<Uid> implements Internal
} }
@Override @Override
public void parse(ParseContext context) throws IOException { public Mapper parse(ParseContext context) throws IOException {
// nothing to do here, we either do it in post parse, or in pre parse. // nothing to do here, we either do it in post parse, or in pre parse.
return null;
} }
@Override @Override

3
src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java
View File

@ -113,8 +113,9 @@ public class VersionFieldMapper extends AbstractFieldMapper<Long> implements Int
} }
@Override @Override
public void parse(ParseContext context) throws IOException { public Mapper parse(ParseContext context) throws IOException {
// _version added in preparse // _version added in preparse
return null;
} }
@Override @Override

523
src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.index.mapper.object; package org.elasticsearch.index.mapper.object;
import com.google.common.collect.Iterables; import com.google.common.collect.Iterables;
import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term; import org.apache.lucene.index.Term;
@ -38,6 +39,7 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.ContentPath;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.DocumentMapperParser;
import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMapperListener; import org.elasticsearch.index.mapper.FieldMapperListener;
@ -45,6 +47,7 @@ import org.elasticsearch.index.mapper.InternalMapper;
import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperBuilders; import org.elasticsearch.index.mapper.MapperBuilders;
import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MapperUtils;
import org.elasticsearch.index.mapper.MergeContext; import org.elasticsearch.index.mapper.MergeContext;
import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.mapper.ObjectMapperListener; import org.elasticsearch.index.mapper.ObjectMapperListener;
@ -84,7 +87,7 @@ import static org.elasticsearch.index.mapper.core.TypeParsers.parsePathType;
/** /**
* *
*/ */
public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll { public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll, Cloneable {
public static final String CONTENT_TYPE = "object"; public static final String CONTENT_TYPE = "object";
public static final String NESTED_CONTENT_TYPE = "nested"; public static final String NESTED_CONTENT_TYPE = "nested";
@ -370,8 +373,6 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
private volatile CopyOnWriteHashMap<String, Mapper> mappers; private volatile CopyOnWriteHashMap<String, Mapper> mappers;
private final Object mutex = new Object();
ObjectMapper(String name, String fullPath, boolean enabled, Nested nested, Dynamic dynamic, ContentPath.Type pathType, Map<String, Mapper> mappers) { ObjectMapper(String name, String fullPath, boolean enabled, Nested nested, Dynamic dynamic, ContentPath.Type pathType, Map<String, Mapper> mappers) {
this.name = name; this.name = name;
this.fullPath = fullPath; this.fullPath = fullPath;
@ -389,6 +390,28 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
this.nestedTypeFilter = new TermFilter(new Term(TypeFieldMapper.NAME, nestedTypePathAsBytes)); this.nestedTypeFilter = new TermFilter(new Term(TypeFieldMapper.NAME, nestedTypePathAsBytes));
} }
@Override
protected ObjectMapper clone() {
ObjectMapper clone;
try {
clone = (ObjectMapper) super.clone();
} catch (CloneNotSupportedException e) {
throw new RuntimeException();
}
return clone;
}
/**
* Build a mapping update with the provided sub mapping update.
*/
public ObjectMapper mappingUpdate(Mapper mapper) {
ObjectMapper mappingUpdate = clone();
// reset the sub mappers
mappingUpdate.mappers = new CopyOnWriteHashMap<>();
mappingUpdate.putMapper(mapper);
return mappingUpdate;
}
@Override @Override
public String name() { public String name() {
return this.name; return this.name;
@ -440,14 +463,16 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
return this.nestedTypeFilter; return this.nestedTypeFilter;
} }
public ObjectMapper putMapper(Mapper mapper) { /**
* Put a new mapper.
* NOTE: this method must be called under the current {@link DocumentMapper}
* lock if concurrent updates are expected.
*/
public void putMapper(Mapper mapper) {
if (mapper instanceof AllFieldMapper.IncludeInAll) { if (mapper instanceof AllFieldMapper.IncludeInAll) {
((AllFieldMapper.IncludeInAll) mapper).includeInAllIfNotSet(includeInAll); ((AllFieldMapper.IncludeInAll) mapper).includeInAllIfNotSet(includeInAll);
} }
synchronized (mutex) { mappers = mappers.copyAndPut(mapper.name(), mapper);
mappers = mappers.copyAndPut(mapper.name(), mapper);
}
return this;
} }
@Override @Override
@ -482,10 +507,10 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
} }
@Override @Override
public void parse(ParseContext context) throws IOException { public ObjectMapper parse(ParseContext context) throws IOException {
if (!enabled) { if (!enabled) {
context.parser().skipChildren(); context.parser().skipChildren();
return; return null;
} }
XContentParser parser = context.parser(); XContentParser parser = context.parser();
@ -493,13 +518,13 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
XContentParser.Token token = parser.currentToken(); XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.VALUE_NULL) { if (token == XContentParser.Token.VALUE_NULL) {
// the object is null ("obj1" : null), simply bail // the object is null ("obj1" : null), simply bail
return; return null;
} }
if (token.isValue() && !allowValue()) { if (token.isValue() && !allowValue()) {
// if we are parsing an object but it is just a value, its only allowed on root level parsers with there // if we are parsing an object but it is just a value, its only allowed on root level parsers with there
// is a field name with the same name as the type // is a field name with the same name as the type
throw new MapperParsingException("object mapping for [" + name + "] tried to parse field [" + currentFieldName + "] as object, but found a concrete value", context.mappingsModified()); throw new MapperParsingException("object mapping for [" + name + "] tried to parse field [" + currentFieldName + "] as object, but found a concrete value");
} }
if (nested.isNested()) { if (nested.isNested()) {
@ -533,21 +558,30 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
token = parser.nextToken(); token = parser.nextToken();
} }
ObjectMapper update = null;
while (token != XContentParser.Token.END_OBJECT) { while (token != XContentParser.Token.END_OBJECT) {
ObjectMapper newUpdate = null;
if (token == XContentParser.Token.START_OBJECT) { if (token == XContentParser.Token.START_OBJECT) {
serializeObject(context, currentFieldName); newUpdate = serializeObject(context, currentFieldName);
} else if (token == XContentParser.Token.START_ARRAY) { } else if (token == XContentParser.Token.START_ARRAY) {
serializeArray(context, currentFieldName); newUpdate = serializeArray(context, currentFieldName);
} else if (token == XContentParser.Token.FIELD_NAME) { } else if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName(); currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.VALUE_NULL) { } else if (token == XContentParser.Token.VALUE_NULL) {
serializeNullValue(context, currentFieldName); serializeNullValue(context, currentFieldName);
} else if (token == null) { } else if (token == null) {
throw new MapperParsingException("object mapping for [" + name + "] tried to parse field [" + currentFieldName + "] as object, but got EOF, has a concrete value been provided to it?", context.mappingsModified()); throw new MapperParsingException("object mapping for [" + name + "] tried to parse field [" + currentFieldName + "] as object, but got EOF, has a concrete value been provided to it?");
} else if (token.isValue()) { } else if (token.isValue()) {
serializeValue(context, currentFieldName, token); newUpdate = serializeValue(context, currentFieldName, token);
} }
token = parser.nextToken(); token = parser.nextToken();
if (newUpdate != null) {
if (update == null) {
update = newUpdate;
} else {
MapperUtils.merge(update, newUpdate);
}
}
} }
// restore the enable path flag // restore the enable path flag
context.path().pathType(origPathType); context.path().pathType(origPathType);
@ -577,6 +611,7 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
} }
} }
} }
return update;
} }
private void serializeNullValue(ParseContext context, String lastFieldName) throws IOException { private void serializeNullValue(ParseContext context, String lastFieldName) throws IOException {
@ -585,54 +620,51 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
if (mapper != null) { if (mapper != null) {
if (mapper instanceof FieldMapper) { if (mapper instanceof FieldMapper) {
if (!((FieldMapper) mapper).supportsNullValue()) { if (!((FieldMapper) mapper).supportsNullValue()) {
throw new MapperParsingException("no object mapping found for null value in [" + lastFieldName + "]", context.mappingsModified()); throw new MapperParsingException("no object mapping found for null value in [" + lastFieldName + "]");
} }
} }
mapper.parse(context); mapper.parse(context);
} else if (dynamic == Dynamic.STRICT) { } else if (dynamic == Dynamic.STRICT) {
throw new StrictDynamicMappingException(fullPath, lastFieldName, context.mappingsModified()); throw new StrictDynamicMappingException(fullPath, lastFieldName);
} }
} }
private void serializeObject(final ParseContext context, String currentFieldName) throws IOException { private ObjectMapper serializeObject(final ParseContext context, String currentFieldName) throws IOException {
if (currentFieldName == null) { if (currentFieldName == null) {
throw new MapperParsingException("object mapping [" + name + "] trying to serialize an object with no field associated with it, current value [" + context.parser().textOrNull() + "]", context.mappingsModified()); throw new MapperParsingException("object mapping [" + name + "] trying to serialize an object with no field associated with it, current value [" + context.parser().textOrNull() + "]");
} }
context.path().add(currentFieldName); context.path().add(currentFieldName);
ObjectMapper update = null;
Mapper objectMapper = mappers.get(currentFieldName); Mapper objectMapper = mappers.get(currentFieldName);
if (objectMapper != null) { if (objectMapper != null) {
objectMapper.parse(context); final Mapper subUpdate = objectMapper.parse(context);
if (subUpdate != null) {
// propagate mapping update
update = mappingUpdate(subUpdate);
}
} else { } else {
Dynamic dynamic = this.dynamic; Dynamic dynamic = this.dynamic;
if (dynamic == null) { if (dynamic == null) {
dynamic = context.root().dynamic(); dynamic = context.root().dynamic();
} }
if (dynamic == Dynamic.STRICT) { if (dynamic == Dynamic.STRICT) {
throw new StrictDynamicMappingException(fullPath, currentFieldName, context.mappingsModified()); throw new StrictDynamicMappingException(fullPath, currentFieldName);
} else if (dynamic == Dynamic.TRUE) { } else if (dynamic == Dynamic.TRUE) {
// we sync here just so we won't add it twice. Its not the end of the world // remove the current field name from path, since template search and the object builder add it as well...
// to sync here since next operations will get it before context.path().remove();
synchronized (mutex) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "object");
objectMapper = mappers.get(currentFieldName); if (builder == null) {
if (objectMapper == null) { builder = MapperBuilders.object(currentFieldName).enabled(true).pathType(pathType);
// remove the current field name from path, since template search and the object builder add it as well... // if this is a non root object, then explicitly set the dynamic behavior if set
context.path().remove(); if (!(this instanceof RootObjectMapper) && this.dynamic != Defaults.DYNAMIC) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "object"); ((Builder) builder).dynamic(this.dynamic);
if (builder == null) {
builder = MapperBuilders.object(currentFieldName).enabled(true).pathType(pathType);
// if this is a non root object, then explicitly set the dynamic behavior if set
if (!(this instanceof RootObjectMapper) && this.dynamic != Defaults.DYNAMIC) {
((Builder) builder).dynamic(this.dynamic);
}
}
BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path());
objectMapper = builder.build(builderContext);
putDynamicMapper(context, currentFieldName, objectMapper);
} else {
objectMapper.parse(context);
} }
} }
BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path());
objectMapper = builder.build(builderContext);
context.path().add(currentFieldName);
update = mappingUpdate(MapperUtils.parseAndMergeUpdate(objectMapper, context));
} else { } else {
// not dynamic, read everything up to end object // not dynamic, read everything up to end object
context.parser().skipChildren(); context.parser().skipChildren();
@ -640,9 +672,10 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
} }
context.path().remove(); context.path().remove();
return update;
} }
private void serializeArray(ParseContext context, String lastFieldName) throws IOException { private ObjectMapper serializeArray(ParseContext context, String lastFieldName) throws IOException {
String arrayFieldName = lastFieldName; String arrayFieldName = lastFieldName;
Mapper mapper = mappers.get(lastFieldName); Mapper mapper = mappers.get(lastFieldName);
if (mapper != null) { if (mapper != null) {
@ -650,9 +683,15 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
// expects an array, if so we pass the context straight to the mapper and if not // expects an array, if so we pass the context straight to the mapper and if not
// we serialize the array components // we serialize the array components
if (mapper instanceof ArrayValueMapperParser) { if (mapper instanceof ArrayValueMapperParser) {
mapper.parse(context); final Mapper subUpdate = mapper.parse(context);
if (subUpdate != null) {
// propagate the mapping update
return mappingUpdate(subUpdate);
} else {
return null;
}
} else { } else {
serializeNonDynamicArray(context, lastFieldName, arrayFieldName); return serializeNonDynamicArray(context, lastFieldName, arrayFieldName);
} }
} else { } else {
@ -661,278 +700,217 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
dynamic = context.root().dynamic(); dynamic = context.root().dynamic();
} }
if (dynamic == Dynamic.STRICT) { if (dynamic == Dynamic.STRICT) {
throw new StrictDynamicMappingException(fullPath, arrayFieldName, context.mappingsModified()); throw new StrictDynamicMappingException(fullPath, arrayFieldName);
} else if (dynamic == Dynamic.TRUE) { } else if (dynamic == Dynamic.TRUE) {
// we sync here just so we won't add it twice. Its not the end of the world Mapper.Builder builder = context.root().findTemplateBuilder(context, arrayFieldName, "object");
// to sync here since next operations will get it before if (builder == null) {
synchronized (mutex) { return serializeNonDynamicArray(context, lastFieldName, arrayFieldName);
mapper = mappers.get(arrayFieldName); }
if (mapper == null) { BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path());
Mapper.Builder builder = context.root().findTemplateBuilder(context, arrayFieldName, "object"); mapper = builder.build(builderContext);
if (builder == null) { if (mapper != null && mapper instanceof ArrayValueMapperParser) {
serializeNonDynamicArray(context, lastFieldName, arrayFieldName); context.path().add(arrayFieldName);
return; mapper = MapperUtils.parseAndMergeUpdate(mapper, context);
} return mappingUpdate(mapper);
BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path()); } else {
mapper = builder.build(builderContext); return serializeNonDynamicArray(context, lastFieldName, arrayFieldName);
if (mapper != null && mapper instanceof ArrayValueMapperParser) {
putDynamicMapper(context, arrayFieldName, mapper);
} else {
serializeNonDynamicArray(context, lastFieldName, arrayFieldName);
}
} else {
serializeNonDynamicArray(context, lastFieldName, arrayFieldName);
}
} }
} else { } else {
return serializeNonDynamicArray(context, lastFieldName, arrayFieldName);
serializeNonDynamicArray(context, lastFieldName, arrayFieldName);
} }
} }
} }
private void putDynamicMapper(ParseContext context, String arrayFieldName, Mapper mapper) throws IOException { private ObjectMapper serializeNonDynamicArray(ParseContext context, String lastFieldName, String arrayFieldName) throws IOException {
// ...now re add it
context.path().add(arrayFieldName);
context.setMappingsModified();
if (context.isWithinNewMapper()) {
// within a new mapper, no need to traverse,
// just parse
mapper.parse(context);
} else {
// create a context of new mapper, so we batch
// aggregate all the changes within
// this object mapper once, and traverse all of
// them to add them in a single go
context.setWithinNewMapper();
try {
mapper.parse(context);
FieldMapperListener.Aggregator newFields = new FieldMapperListener.Aggregator();
ObjectMapperListener.Aggregator newObjects = new ObjectMapperListener.Aggregator();
mapper.traverse(newFields);
mapper.traverse(newObjects);
// callback on adding those fields!
context.docMapper().addFieldMappers(newFields.mappers);
context.docMapper().addObjectMappers(newObjects.mappers);
} finally {
context.clearWithinNewMapper();
}
}
// only put after we traversed and did the
// callbacks, so other parsing won't see it only
// after we
// properly traversed it and adding the mappers
putMapper(mapper);
}
private void serializeNonDynamicArray(ParseContext context, String lastFieldName, String arrayFieldName) throws IOException {
XContentParser parser = context.parser(); XContentParser parser = context.parser();
XContentParser.Token token; XContentParser.Token token;
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
if (token == XContentParser.Token.START_OBJECT) { if (token == XContentParser.Token.START_OBJECT) {
serializeObject(context, lastFieldName); return serializeObject(context, lastFieldName);
} else if (token == XContentParser.Token.START_ARRAY) { } else if (token == XContentParser.Token.START_ARRAY) {
serializeArray(context, lastFieldName); return serializeArray(context, lastFieldName);
} else if (token == XContentParser.Token.FIELD_NAME) { } else if (token == XContentParser.Token.FIELD_NAME) {
lastFieldName = parser.currentName(); lastFieldName = parser.currentName();
} else if (token == XContentParser.Token.VALUE_NULL) { } else if (token == XContentParser.Token.VALUE_NULL) {
serializeNullValue(context, lastFieldName); serializeNullValue(context, lastFieldName);
} else if (token == null) { } else if (token == null) {
throw new MapperParsingException("object mapping for [" + name + "] with array for [" + arrayFieldName + "] tried to parse as array, but got EOF, is there a mismatch in types for the same field?", context.mappingsModified()); throw new MapperParsingException("object mapping for [" + name + "] with array for [" + arrayFieldName + "] tried to parse as array, but got EOF, is there a mismatch in types for the same field?");
} else { } else {
serializeValue(context, lastFieldName, token); return serializeValue(context, lastFieldName, token);
} }
} }
return null;
} }
private void serializeValue(final ParseContext context, String currentFieldName, XContentParser.Token token) throws IOException { private ObjectMapper serializeValue(final ParseContext context, String currentFieldName, XContentParser.Token token) throws IOException {
if (currentFieldName == null) { if (currentFieldName == null) {
throw new MapperParsingException("object mapping [" + name + "] trying to serialize a value with no field associated with it, current value [" + context.parser().textOrNull() + "]", context.mappingsModified()); throw new MapperParsingException("object mapping [" + name + "] trying to serialize a value with no field associated with it, current value [" + context.parser().textOrNull() + "]");
} }
Mapper mapper = mappers.get(currentFieldName); Mapper mapper = mappers.get(currentFieldName);
if (mapper != null) { if (mapper != null) {
mapper.parse(context); Mapper subUpdate = mapper.parse(context);
if (subUpdate == null) {
return null;
}
return mappingUpdate(subUpdate);
} else { } else {
parseDynamicValue(context, currentFieldName, token); return parseDynamicValue(context, currentFieldName, token);
} }
} }
public void parseDynamicValue(final ParseContext context, String currentFieldName, XContentParser.Token token) throws IOException { public ObjectMapper parseDynamicValue(final ParseContext context, String currentFieldName, XContentParser.Token token) throws IOException {
Dynamic dynamic = this.dynamic; Dynamic dynamic = this.dynamic;
if (dynamic == null) { if (dynamic == null) {
dynamic = context.root().dynamic(); dynamic = context.root().dynamic();
} }
if (dynamic == Dynamic.STRICT) { if (dynamic == Dynamic.STRICT) {
throw new StrictDynamicMappingException(fullPath, currentFieldName, context.mappingsModified()); throw new StrictDynamicMappingException(fullPath, currentFieldName);
} }
if (dynamic == Dynamic.FALSE) { if (dynamic == Dynamic.FALSE) {
return; return null;
} }
// we sync here since we don't want to add this field twice to the document mapper Mapper mapper = null;
// its not the end of the world, since we add it to the mappers once we create it BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path());
// so next time we won't even get here for this field if (token == XContentParser.Token.VALUE_STRING) {
synchronized (mutex) { boolean resolved = false;
Mapper mapper = mappers.get(currentFieldName);
if (mapper == null) {
BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path());
if (token == XContentParser.Token.VALUE_STRING) {
boolean resolved = false;
// do a quick test to see if its fits a dynamic template, if so, use it. // do a quick test to see if its fits a dynamic template, if so, use it.
// we need to do it here so we can handle things like attachment templates, where calling // we need to do it here so we can handle things like attachment templates, where calling
// text (to see if its a date) causes the binary value to be cleared // text (to see if its a date) causes the binary value to be cleared
if (!resolved) { if (!resolved) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "string", null); Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "string", null);
if (builder != null) { if (builder != null) {
mapper = builder.build(builderContext); mapper = builder.build(builderContext);
resolved = true; resolved = true;
} }
} }
if (!resolved && context.root().dateDetection()) { if (!resolved && context.root().dateDetection()) {
String text = context.parser().text(); String text = context.parser().text();
// a safe check since "1" gets parsed as well // a safe check since "1" gets parsed as well
if (Strings.countOccurrencesOf(text, ":") > 1 || Strings.countOccurrencesOf(text, "-") > 1 || Strings.countOccurrencesOf(text, "/") > 1) { if (Strings.countOccurrencesOf(text, ":") > 1 || Strings.countOccurrencesOf(text, "-") > 1 || Strings.countOccurrencesOf(text, "/") > 1) {
for (FormatDateTimeFormatter dateTimeFormatter : context.root().dynamicDateTimeFormatters()) { for (FormatDateTimeFormatter dateTimeFormatter : context.root().dynamicDateTimeFormatters()) {
try {
dateTimeFormatter.parser().parseMillis(text);
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "date");
if (builder == null) {
builder = dateField(currentFieldName).dateTimeFormatter(dateTimeFormatter);
}
mapper = builder.build(builderContext);
resolved = true;
break;
} catch (Exception e) {
// failure to parse this, continue
}
}
}
}
if (!resolved && context.root().numericDetection()) {
String text = context.parser().text();
try { try {
Long.parseLong(text); dateTimeFormatter.parser().parseMillis(text);
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long"); Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "date");
if (builder == null) { if (builder == null) {
builder = longField(currentFieldName); builder = dateField(currentFieldName).dateTimeFormatter(dateTimeFormatter);
} }
mapper = builder.build(builderContext); mapper = builder.build(builderContext);
resolved = true; resolved = true;
break;
} catch (Exception e) { } catch (Exception e) {
// not a long number // failure to parse this, continue
}
if (!resolved) {
try {
Double.parseDouble(text);
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double");
if (builder == null) {
builder = doubleField(currentFieldName);
}
mapper = builder.build(builderContext);
resolved = true;
} catch (Exception e) {
// not a long number
}
} }
} }
if (!resolved) { }
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "string"); }
if (builder == null) { if (!resolved && context.root().numericDetection()) {
builder = stringField(currentFieldName); String text = context.parser().text();
} try {
mapper = builder.build(builderContext); Long.parseLong(text);
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long");
if (builder == null) {
builder = longField(currentFieldName);
} }
} else if (token == XContentParser.Token.VALUE_NUMBER) { mapper = builder.build(builderContext);
XContentParser.NumberType numberType = context.parser().numberType(); resolved = true;
if (numberType == XContentParser.NumberType.INT) { } catch (Exception e) {
if (context.parser().estimatedNumberType()) { // not a long number
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long"); }
if (builder == null) { if (!resolved) {
builder = longField(currentFieldName); try {
} Double.parseDouble(text);
mapper = builder.build(builderContext);
} else {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "integer");
if (builder == null) {
builder = integerField(currentFieldName);
}
mapper = builder.build(builderContext);
}
} else if (numberType == XContentParser.NumberType.LONG) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long");
if (builder == null) {
builder = longField(currentFieldName);
}
mapper = builder.build(builderContext);
} else if (numberType == XContentParser.NumberType.FLOAT) {
if (context.parser().estimatedNumberType()) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double");
if (builder == null) {
builder = doubleField(currentFieldName);
}
mapper = builder.build(builderContext);
} else {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "float");
if (builder == null) {
builder = floatField(currentFieldName);
}
mapper = builder.build(builderContext);
}
} else if (numberType == XContentParser.NumberType.DOUBLE) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double"); Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double");
if (builder == null) { if (builder == null) {
builder = doubleField(currentFieldName); builder = doubleField(currentFieldName);
} }
mapper = builder.build(builderContext); mapper = builder.build(builderContext);
resolved = true;
} catch (Exception e) {
// not a long number
} }
} else if (token == XContentParser.Token.VALUE_BOOLEAN) { }
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "boolean"); }
if (!resolved) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "string");
if (builder == null) {
builder = stringField(currentFieldName);
}
mapper = builder.build(builderContext);
}
} else if (token == XContentParser.Token.VALUE_NUMBER) {
XContentParser.NumberType numberType = context.parser().numberType();
if (numberType == XContentParser.NumberType.INT) {
if (context.parser().estimatedNumberType()) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long");
if (builder == null) { if (builder == null) {
builder = booleanField(currentFieldName); builder = longField(currentFieldName);
}
mapper = builder.build(builderContext);
} else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "binary");
if (builder == null) {
builder = binaryField(currentFieldName);
} }
mapper = builder.build(builderContext); mapper = builder.build(builderContext);
} else { } else {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, null); Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "integer");
if (builder != null) { if (builder == null) {
mapper = builder.build(builderContext); builder = integerField(currentFieldName);
} else {
// TODO how do we identify dynamically that its a binary value?
throw new ElasticsearchIllegalStateException("Can't handle serializing a dynamic type with content token [" + token + "] and field name [" + currentFieldName + "]");
} }
mapper = builder.build(builderContext);
} }
} else if (numberType == XContentParser.NumberType.LONG) {
if (context.isWithinNewMapper()) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long");
mapper.parse(context); if (builder == null) {
builder = longField(currentFieldName);
}
mapper = builder.build(builderContext);
} else if (numberType == XContentParser.NumberType.FLOAT) {
if (context.parser().estimatedNumberType()) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double");
if (builder == null) {
builder = doubleField(currentFieldName);
}
mapper = builder.build(builderContext);
} else { } else {
context.setWithinNewMapper(); Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "float");
try { if (builder == null) {
mapper.parse(context); builder = floatField(currentFieldName);
FieldMapperListener.Aggregator newFields = new FieldMapperListener.Aggregator();
mapper.traverse(newFields);
context.docMapper().addFieldMappers(newFields.mappers);
} finally {
context.clearWithinNewMapper();
} }
mapper = builder.build(builderContext);
} }
} else if (numberType == XContentParser.NumberType.DOUBLE) {
// only put after we traversed and did the callbacks, so other parsing won't see it only after we Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double");
// properly traversed it and adding the mappers if (builder == null) {
putMapper(mapper); builder = doubleField(currentFieldName);
context.setMappingsModified(); }
mapper = builder.build(builderContext);
}
} else if (token == XContentParser.Token.VALUE_BOOLEAN) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "boolean");
if (builder == null) {
builder = booleanField(currentFieldName);
}
mapper = builder.build(builderContext);
} else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "binary");
if (builder == null) {
builder = binaryField(currentFieldName);
}
mapper = builder.build(builderContext);
} else {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, null);
if (builder != null) {
mapper = builder.build(builderContext);
} else { } else {
mapper.parse(context); // TODO how do we identify dynamically that its a binary value?
throw new ElasticsearchIllegalStateException("Can't handle serializing a dynamic type with content token [" + token + "] and field name [" + currentFieldName + "]");
} }
} }
mapper = MapperUtils.parseAndMergeUpdate(mapper, context);
ObjectMapper update = null;
if (mapper != null) {
update = mappingUpdate(mapper);
}
return update;
} }
@Override @Override
@ -966,33 +944,30 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
List<Mapper> mappersToPut = new ArrayList<>(); List<Mapper> mappersToPut = new ArrayList<>();
FieldMapperListener.Aggregator newFieldMappers = new FieldMapperListener.Aggregator(); FieldMapperListener.Aggregator newFieldMappers = new FieldMapperListener.Aggregator();
ObjectMapperListener.Aggregator newObjectMappers = new ObjectMapperListener.Aggregator(); ObjectMapperListener.Aggregator newObjectMappers = new ObjectMapperListener.Aggregator();
synchronized (mutex) { for (Mapper mapper : mergeWithObject.mappers.values()) {
for (Mapper mapper : mergeWithObject.mappers.values()) { Mapper mergeWithMapper = mapper;
Mapper mergeWithMapper = mapper; Mapper mergeIntoMapper = mappers.get(mergeWithMapper.name());
Mapper mergeIntoMapper = mappers.get(mergeWithMapper.name()); if (mergeIntoMapper == null) {
if (mergeIntoMapper == null) { // no mapping, simply add it if not simulating
// no mapping, simply add it if not simulating if (!mergeContext.mergeFlags().simulate()) {
if (!mergeContext.mergeFlags().simulate()) { mappersToPut.add(mergeWithMapper);
mappersToPut.add(mergeWithMapper); mergeWithMapper.traverse(newFieldMappers);
mergeWithMapper.traverse(newFieldMappers); mergeWithMapper.traverse(newObjectMappers);
mergeWithMapper.traverse(newObjectMappers);
}
} else {
mergeIntoMapper.merge(mergeWithMapper, mergeContext);
} }
} } else {
if (!newFieldMappers.mappers.isEmpty()) { mergeIntoMapper.merge(mergeWithMapper, mergeContext);
mergeContext.docMapper().addFieldMappers(newFieldMappers.mappers);
}
if (!newObjectMappers.mappers.isEmpty()) {
mergeContext.docMapper().addObjectMappers(newObjectMappers.mappers);
}
// and the mappers only after the administration have been done, so it will not be visible to parser (which first try to read with no lock)
for (Mapper mapper : mappersToPut) {
putMapper(mapper);
} }
} }
if (!newFieldMappers.mappers.isEmpty()) {
mergeContext.addFieldMappers(newFieldMappers.mappers);
}
if (!newObjectMappers.mappers.isEmpty()) {
mergeContext.addObjectMappers(newObjectMappers.mappers);
}
// add the mappers only after the administration have been done, so it will not be visible to parser (which first try to read with no lock)
for (Mapper mapper : mappersToPut) {
putMapper(mapper);
}
} }
protected void doMerge(ObjectMapper mergeWith, MergeContext mergeContext) { protected void doMerge(ObjectMapper mergeWith, MergeContext mergeContext) {

11
src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper.object;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings; import org.elasticsearch.common.Strings;
import org.elasticsearch.common.joda.FormatDateTimeFormatter; import org.elasticsearch.common.joda.FormatDateTimeFormatter;
@ -206,6 +207,14 @@ public class RootObjectMapper extends ObjectMapper {
this.numericDetection = numericDetection; this.numericDetection = numericDetection;
} }
@Override
public ObjectMapper mappingUpdate(Mapper mapper) {
RootObjectMapper update = (RootObjectMapper) super.mappingUpdate(mapper);
// dynamic templates are irrelevant for dynamic mappings updates
update.dynamicTemplates = new DynamicTemplate[0];
return update;
}
public boolean dateDetection() { public boolean dateDetection() {
return this.dateDetection; return this.dateDetection;
} }
@ -231,7 +240,7 @@ public class RootObjectMapper extends ObjectMapper {
String mappingType = dynamicTemplate.mappingType(dynamicType); String mappingType = dynamicTemplate.mappingType(dynamicType);
Mapper.TypeParser typeParser = parserContext.typeParser(mappingType); Mapper.TypeParser typeParser = parserContext.typeParser(mappingType);
if (typeParser == null) { if (typeParser == null) {
throw new MapperParsingException("failed to find type parsed [" + mappingType + "] for [" + name + "]", context.mappingsModified()); throw new MapperParsingException("failed to find type parsed [" + mappingType + "] for [" + name + "]");
} }
return typeParser.parse(name, dynamicTemplate.mappingForName(name, dynamicType), parserContext); return typeParser.parse(name, dynamicTemplate.mappingForName(name, dynamicType), parserContext);
} }

4
src/main/java/org/elasticsearch/index/shard/IndexShard.java
View File

@ -457,7 +457,7 @@ public class IndexShard extends AbstractIndexShardComponent {
ParsedDocument doc = docMapper.v1().parse(source).setMappingsModified(docMapper); ParsedDocument doc = docMapper.v1().parse(source).setMappingsModified(docMapper);
return new Engine.Create(docMapper.v1(), docMapper.v1().uidMapper().term(doc.uid().stringValue()), doc, version, versionType, origin, startTime, canHaveDuplicates, autoGeneratedId); return new Engine.Create(docMapper.v1(), docMapper.v1().uidMapper().term(doc.uid().stringValue()), doc, version, versionType, origin, startTime, canHaveDuplicates, autoGeneratedId);
} catch (Throwable t) { } catch (Throwable t) {
if (docMapper.v2() || (t instanceof MapperParsingException && ((MapperParsingException)t).isMappingsModified())) { if (docMapper.v2()) {
throw new WriteFailureException(t, docMapper.v1().type()); throw new WriteFailureException(t, docMapper.v1().type());
} else { } else {
throw t; throw t;
@ -493,7 +493,7 @@ public class IndexShard extends AbstractIndexShardComponent {
ParsedDocument doc = docMapper.v1().parse(source).setMappingsModified(docMapper); ParsedDocument doc = docMapper.v1().parse(source).setMappingsModified(docMapper);
return new Engine.Index(docMapper.v1(), docMapper.v1().uidMapper().term(doc.uid().stringValue()), doc, version, versionType, origin, startTime, canHaveDuplicates); return new Engine.Index(docMapper.v1(), docMapper.v1().uidMapper().term(doc.uid().stringValue()), doc, version, versionType, origin, startTime, canHaveDuplicates);
} catch (Throwable t) { } catch (Throwable t) {
if (docMapper.v2() || (t instanceof MapperParsingException && ((MapperParsingException) t).isMappingsModified())) { if (docMapper.v2()) {
throw new WriteFailureException(t, docMapper.v1().type()); throw new WriteFailureException(t, docMapper.v1().type());
} else { } else {
throw t; throw t;

56
src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
View File

@ -37,23 +37,38 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.engine.Engine.Searcher;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.DocumentMapperParser;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.mapper.ParseContext.Document;
import org.elasticsearch.index.mapper.internal.IndexFieldMapper; import org.elasticsearch.index.mapper.internal.IndexFieldMapper;
import org.elasticsearch.index.mapper.internal.SizeFieldMapper; import org.elasticsearch.index.mapper.internal.SizeFieldMapper;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper; import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
import org.elasticsearch.test.ElasticsearchSingleNodeTest; import org.elasticsearch.test.ElasticsearchSingleNodeTest;
import org.hamcrest.Matchers; import org.hamcrest.Matchers;
import org.junit.Test; import org.junit.Test;
import java.io.IOException; import java.io.IOException;
import java.util.*; import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.common.io.Streams.copyToBytesFromClasspath; import static org.elasticsearch.common.io.Streams.copyToBytesFromClasspath;
import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath; import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.hamcrest.Matchers.*; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.nullValue;
/** /**
* *
@ -82,7 +97,8 @@ public class SimpleAllMapperTests extends ElasticsearchSingleNodeTest {
@Test @Test
public void testAllMappersNoBoost() throws Exception { public void testAllMappersNoBoost() throws Exception {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/noboost-mapping.json"); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/noboost-mapping.json");
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); IndexService index = createIndex("test");
DocumentMapper docMapper = index.mapperService().documentMapperParser().parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json"); byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = docMapper.parse(new BytesArray(json)).rootDoc(); Document doc = docMapper.parse(new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all"); AllField field = (AllField) doc.getField("_all");
@ -93,7 +109,6 @@ public class SimpleAllMapperTests extends ElasticsearchSingleNodeTest {
assertThat(allEntries.fields().contains("simple1"), equalTo(true)); assertThat(allEntries.fields().contains("simple1"), equalTo(true));
FieldMapper mapper = docMapper.mappers().smartNameFieldMapper("_all"); FieldMapper mapper = docMapper.mappers().smartNameFieldMapper("_all");
assertThat(field.fieldType().omitNorms(), equalTo(false)); assertThat(field.fieldType().omitNorms(), equalTo(false));
assertThat(mapper.queryStringTermQuery(new Term("_all", "foobar")), Matchers.instanceOf(TermQuery.class));
} }
@Test @Test
@ -110,7 +125,7 @@ public class SimpleAllMapperTests extends ElasticsearchSingleNodeTest {
assertThat(allEntries.fields().contains("simple1"), equalTo(true)); assertThat(allEntries.fields().contains("simple1"), equalTo(true));
FieldMapper mapper = docMapper.mappers().smartNameFieldMapper("_all"); FieldMapper mapper = docMapper.mappers().smartNameFieldMapper("_all");
assertThat(field.fieldType().omitNorms(), equalTo(false)); assertThat(field.fieldType().omitNorms(), equalTo(false));
assertThat(mapper.queryStringTermQuery(new Term("_all", "foobar")), Matchers.instanceOf(TermQuery.class)); assertThat(mapper.queryStringTermQuery(new Term("_all", "foobar")), Matchers.instanceOf(AllTermQuery.class));
} }
@ -223,7 +238,6 @@ public class SimpleAllMapperTests extends ElasticsearchSingleNodeTest {
boolean omitNorms = false; boolean omitNorms = false;
boolean stored = false; boolean stored = false;
boolean enabled = true; boolean enabled = true;
boolean autoBoost = false;
boolean tv_stored = false; boolean tv_stored = false;
boolean tv_payloads = false; boolean tv_payloads = false;
boolean tv_offsets = false; boolean tv_offsets = false;
@ -249,9 +263,6 @@ public class SimpleAllMapperTests extends ElasticsearchSingleNodeTest {
if (randomBoolean()) { if (randomBoolean()) {
booleanOptionList.add(new Tuple<>("enabled", enabled = randomBoolean())); booleanOptionList.add(new Tuple<>("enabled", enabled = randomBoolean()));
} }
if (randomBoolean()) {
booleanOptionList.add(new Tuple<>("auto_boost", autoBoost = randomBoolean()));
}
if (randomBoolean()) { if (randomBoolean()) {
booleanOptionList.add(new Tuple<>("store_term_vector_offsets", tv_offsets = randomBoolean())); booleanOptionList.add(new Tuple<>("store_term_vector_offsets", tv_offsets = randomBoolean()));
} }
@ -312,14 +323,6 @@ public class SimpleAllMapperTests extends ElasticsearchSingleNodeTest {
} else { } else {
assertThat(field, nullValue()); assertThat(field, nullValue());
} }
Term term = new Term("foo", "bar");
Query query = builtDocMapper.allFieldMapper().queryStringTermQuery(term);
if (autoBoost) {
assertThat(query, equalTo((Query)new AllTermQuery(term)));
} else {
assertThat(query, equalTo((Query)new TermQuery(term)));
}
if (similarity == null || similarity.equals("TF/IDF")) { if (similarity == null || similarity.equals("TF/IDF")) {
assertThat(builtDocMapper.allFieldMapper().similarity(), nullValue()); assertThat(builtDocMapper.allFieldMapper().similarity(), nullValue());
} else { } else {
@ -458,4 +461,19 @@ public class SimpleAllMapperTests extends ElasticsearchSingleNodeTest {
assertThat(e.getDetailedMessage(), containsString("[_all] is always tokenized and cannot have doc values")); assertThat(e.getDetailedMessage(), containsString("[_all] is always tokenized and cannot have doc values"));
} }
} }
public void testAutoBoost() throws Exception {
for (boolean boost : new boolean[] {false, true}) {
String index = "test_" + boost;
IndexService indexService = createIndex(index, client().admin().indices().prepareCreate(index).addMapping("type", "foo", "type=string" + (boost ? ",boost=2" : "")));
client().prepareIndex(index, "type").setSource("foo", "bar").get();
client().admin().indices().prepareRefresh(index).get();
Query query = indexService.mapperService().documentMapper("type").allFieldMapper().termQuery("bar", null);
try (Searcher searcher = indexService.shard(0).acquireSearcher("tests")) {
query = searcher.searcher().rewrite(query);
final Class<?> expected = boost ? AllTermQuery.class : TermQuery.class;
assertThat(query, Matchers.instanceOf(expected));
}
}
}
} }

202
src/test/java/org/elasticsearch/index/mapper/dynamic/DynamicMappingTests.java
View File

@ -19,15 +19,31 @@
package org.elasticsearch.index.mapper.dynamic; package org.elasticsearch.index.mapper.dynamic;
import com.google.common.base.Predicate; import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.ContentPath;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.DocumentMapperParser;
import org.elasticsearch.index.mapper.FieldMappers;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.index.mapper.StrictDynamicMappingException;
import org.elasticsearch.test.ElasticsearchSingleNodeTest; import org.elasticsearch.test.ElasticsearchSingleNodeTest;
import org.junit.Test;
import java.io.IOException; import java.io.IOException;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
@ -39,7 +55,6 @@ import static org.hamcrest.Matchers.nullValue;
public class DynamicMappingTests extends ElasticsearchSingleNodeTest { public class DynamicMappingTests extends ElasticsearchSingleNodeTest {
@Test
public void testDynamicTrue() throws IOException { public void testDynamicTrue() throws IOException {
String mapping = jsonBuilder().startObject().startObject("type") String mapping = jsonBuilder().startObject().startObject("type")
.field("dynamic", "true") .field("dynamic", "true")
@ -60,7 +75,6 @@ public class DynamicMappingTests extends ElasticsearchSingleNodeTest {
assertThat(doc.rootDoc().get("field2"), equalTo("value2")); assertThat(doc.rootDoc().get("field2"), equalTo("value2"));
} }
@Test
public void testDynamicFalse() throws IOException { public void testDynamicFalse() throws IOException {
String mapping = jsonBuilder().startObject().startObject("type") String mapping = jsonBuilder().startObject().startObject("type")
.field("dynamic", "false") .field("dynamic", "false")
@ -82,7 +96,6 @@ public class DynamicMappingTests extends ElasticsearchSingleNodeTest {
} }
@Test
public void testDynamicStrict() throws IOException { public void testDynamicStrict() throws IOException {
String mapping = jsonBuilder().startObject().startObject("type") String mapping = jsonBuilder().startObject().startObject("type")
.field("dynamic", "strict") .field("dynamic", "strict")
@ -116,7 +129,6 @@ public class DynamicMappingTests extends ElasticsearchSingleNodeTest {
} }
} }
@Test
public void testDynamicFalseWithInnerObjectButDynamicSetOnRoot() throws IOException { public void testDynamicFalseWithInnerObjectButDynamicSetOnRoot() throws IOException {
String mapping = jsonBuilder().startObject().startObject("type") String mapping = jsonBuilder().startObject().startObject("type")
.field("dynamic", "false") .field("dynamic", "false")
@ -140,7 +152,6 @@ public class DynamicMappingTests extends ElasticsearchSingleNodeTest {
assertThat(doc.rootDoc().get("obj1.field2"), nullValue()); assertThat(doc.rootDoc().get("obj1.field2"), nullValue());
} }
@Test
public void testDynamicStrictWithInnerObjectButDynamicSetOnRoot() throws IOException { public void testDynamicStrictWithInnerObjectButDynamicSetOnRoot() throws IOException {
String mapping = jsonBuilder().startObject().startObject("type") String mapping = jsonBuilder().startObject().startObject("type")
.field("dynamic", "strict") .field("dynamic", "strict")
@ -173,7 +184,6 @@ public class DynamicMappingTests extends ElasticsearchSingleNodeTest {
assertTrue(mappers != null && mappers.isEmpty() == false); assertTrue(mappers != null && mappers.isEmpty() == false);
} }
@Test
public void testIndexingFailureDoesStillCreateType() throws IOException, InterruptedException { public void testIndexingFailureDoesStillCreateType() throws IOException, InterruptedException {
XContentBuilder mapping = jsonBuilder().startObject().startObject("_default_") XContentBuilder mapping = jsonBuilder().startObject().startObject("_default_")
.field("dynamic", "strict") .field("dynamic", "strict")
@ -202,7 +212,6 @@ public class DynamicMappingTests extends ElasticsearchSingleNodeTest {
} }
@Test
public void testTypeCreatedProperly() throws IOException, InterruptedException { public void testTypeCreatedProperly() throws IOException, InterruptedException {
XContentBuilder mapping = jsonBuilder().startObject().startObject("_default_") XContentBuilder mapping = jsonBuilder().startObject().startObject("_default_")
.field("dynamic", "strict") .field("dynamic", "strict")
@ -243,7 +252,6 @@ public class DynamicMappingTests extends ElasticsearchSingleNodeTest {
assertNotNull(getMappingsResponse.getMappings().get("test").get("type")); assertNotNull(getMappingsResponse.getMappings().get("test").get("type"));
} }
@Test
public void testFieldsCreatedWithPartialParsing() throws IOException, InterruptedException { public void testFieldsCreatedWithPartialParsing() throws IOException, InterruptedException {
XContentBuilder mapping = jsonBuilder().startObject().startObject("doc") XContentBuilder mapping = jsonBuilder().startObject().startObject("doc")
.startObject("properties") .startObject("properties")
@ -304,4 +312,178 @@ public class DynamicMappingTests extends ElasticsearchSingleNodeTest {
} }
})); }));
} }
private String serialize(ToXContent mapper) throws Exception {
XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
mapper.toXContent(builder, new ToXContent.MapParams(ImmutableMap.<String, String>of()));
return builder.endObject().string();
}
private Mapper parse(DocumentMapper mapper, DocumentMapperParser parser, XContentBuilder builder) throws Exception {
Settings settings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
ParseContext.InternalParseContext ctx = new ParseContext.InternalParseContext("test", settings, parser, mapper, new ContentPath(0));
SourceToParse source = SourceToParse.source(builder.bytes());
ctx.reset(XContentHelper.createParser(source.source()), new ParseContext.Document(), source, null);
assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken());
ctx.parser().nextToken();
return mapper.root().parse(ctx);
}
public void testDynamicMappingsNotNeeded() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("foo").field("type", "string").endObject().endObject()
.endObject().string();
DocumentMapper mapper = parser.parse(mapping);
Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject());
// foo is already defined in the mappings
assertNull(update);
}
public void testField() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").endObject().endObject()
.endObject().string();
DocumentMapper mapper = parser.parse(mapping);
assertEquals(mapping, serialize(mapper));
Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject());
assertNotNull(update);
// original mapping not modified
assertEquals(mapping, serialize(mapper));
// but we have an update
assertEquals("{\"type\":{\"properties\":{\"foo\":{\"type\":\"string\"}}}}", serialize(update));
}
public void testIncremental() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
// Make sure that mapping updates are incremental, this is important for performance otherwise
// every new field introduction runs in linear time with the total number of fields
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("foo").field("type", "string").endObject().endObject()
.endObject().string();
DocumentMapper mapper = parser.parse(mapping);
assertEquals(mapping, serialize(mapper));
Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar").field("bar", "baz").endObject());
assertNotNull(update);
// original mapping not modified
assertEquals(mapping, serialize(mapper));
// but we have an update
assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
// foo is NOT in the update
.startObject("bar").field("type", "string").endObject()
.endObject().endObject().string(), serialize(update));
}
public void testIntroduceTwoFields() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").endObject().endObject()
.endObject().string();
DocumentMapper mapper = parser.parse(mapping);
assertEquals(mapping, serialize(mapper));
Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar").field("bar", "baz").endObject());
assertNotNull(update);
// original mapping not modified
assertEquals(mapping, serialize(mapper));
// but we have an update
assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("bar").field("type", "string").endObject()
.startObject("foo").field("type", "string").endObject()
.endObject().endObject().string(), serialize(update));
}
public void testObject() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").endObject().endObject()
.endObject().string();
DocumentMapper mapper = parser.parse(mapping);
assertEquals(mapping, serialize(mapper));
Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startObject("foo").startObject("bar").field("baz", "foo").endObject().endObject().endObject());
assertNotNull(update);
// original mapping not modified
assertEquals(mapping, serialize(mapper));
// but we have an update
assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("foo").startObject("properties").startObject("bar").startObject("properties").startObject("baz").field("type", "string").endObject().endObject().endObject().endObject().endObject()
.endObject().endObject().endObject().string(), serialize(update));
}
public void testArray() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").endObject().endObject()
.endObject().string();
DocumentMapper mapper = parser.parse(mapping);
assertEquals(mapping, serialize(mapper));
Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startArray("foo").value("bar").value("baz").endArray().endObject());
assertNotNull(update);
// original mapping not modified
assertEquals(mapping, serialize(mapper));
// but we have an update
assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("foo").field("type", "string").endObject()
.endObject().endObject().endObject().string(), serialize(update));
}
public void testInnerDynamicMapping() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties")
.startObject("foo").field("type", "object").endObject()
.endObject().endObject().endObject().string();
DocumentMapper mapper = parser.parse(mapping);
assertEquals(mapping, serialize(mapper));
Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startObject("foo").startObject("bar").field("baz", "foo").endObject().endObject().endObject());
assertNotNull(update);
// original mapping not modified
assertEquals(mapping, serialize(mapper));
// but we have an update
assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("foo").startObject("properties").startObject("bar").startObject("properties").startObject("baz").field("type", "string").endObject().endObject().endObject().endObject().endObject()
.endObject().endObject().endObject().string(), serialize(update));
}
public void testComplexArray() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").endObject().endObject()
.endObject().string();
DocumentMapper mapper = parser.parse(mapping);
assertEquals(mapping, serialize(mapper));
Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startArray("foo")
.startObject().field("bar", "baz").endObject()
.startObject().field("baz", 3).endObject()
.endArray().endObject());
assertEquals(mapping, serialize(mapper));
assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("foo").startObject("properties")
.startObject("bar").field("type", "string").endObject()
.startObject("baz").field("type", "long").endObject()
.endObject().endObject()
.endObject().endObject().endObject().string(), serialize(update));
}
} }

3
src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java
View File

@ -185,7 +185,7 @@ public class ExternalMapper extends AbstractFieldMapper<Object> {
} }
@Override @Override
public void parse(ParseContext context) throws IOException { public Mapper parse(ParseContext context) throws IOException {
byte[] bytes = "Hello world".getBytes(Charset.defaultCharset()); byte[] bytes = "Hello world".getBytes(Charset.defaultCharset());
binMapper.parse(context.createExternalValueContext(bytes)); binMapper.parse(context.createExternalValueContext(bytes));
@ -210,6 +210,7 @@ public class ExternalMapper extends AbstractFieldMapper<Object> {
if (copyTo != null) { if (copyTo != null) {
copyTo.parse(context); copyTo.parse(context);
} }
return null;
} }
@Override @Override

3
src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalRootMapper.java
View File

@ -39,7 +39,8 @@ public class ExternalRootMapper implements RootMapper {
} }
@Override @Override
public void parse(ParseContext context) throws IOException { public Mapper parse(ParseContext context) throws IOException {
return null;
} }
@Override @Override

2
src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterTests.java
View File

@ -68,7 +68,7 @@ public class UpdateMappingOnClusterTests extends ElasticsearchIntegrationTest {
"[_all] has different store_term_vector_payloads values", "[_all] has different store_term_vector_payloads values",
"[_all] has different analyzer", "[_all] has different analyzer",
"[_all] has different similarity"}; "[_all] has different similarity"};
// auto_boost and fielddata and search_analyzer should not report conflict // fielddata and search_analyzer should not report conflict
testConflict(mapping, mappingUpdate, errorMessage); testConflict(mapping, mappingUpdate, errorMessage);
} }

1
src/test/java/org/elasticsearch/index/mapper/update/all_mapping_create_index.json
View File

@ -2,7 +2,6 @@
"mappings": { "mappings": {
"type": { "type": {
"_all": { "_all": {
"auto_boost": true,
"store": true, "store": true,
"store_term_vectors": true, "store_term_vectors": true,
"store_term_vector_offsets": true, "store_term_vector_offsets": true,

1
src/test/java/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json
View File

@ -1,7 +1,6 @@
{ {
"type": { "type": {
"_all": { "_all": {
"auto_boost": false,
"store": false, "store": false,
"enabled": false, "enabled": false,
"store_term_vectors": false, "store_term_vectors": false,

5
src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java
View File

@ -372,11 +372,6 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
.field("enabled", randomBoolean()) .field("enabled", randomBoolean())
.endObject(); .endObject();
} }
if (randomBoolean()) {
mappings.startObject(AllFieldMapper.NAME)
.field("auto_boost", true)
.endObject();
}
if (randomBoolean()) { if (randomBoolean()) {
mappings.startObject(SourceFieldMapper.NAME) mappings.startObject(SourceFieldMapper.NAME)
.field("compress", randomBoolean()) .field("compress", randomBoolean())

BIN
src/test/resources/org/elasticsearch/bwcompat/index-1.5.0.zip
View File

Binary file not shown.

BIN
src/test/resources/org/elasticsearch/bwcompat/repo-1.5.0.zip
View File

Binary file not shown.