honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Migrate CompletionFieldMapper to parametrized format (#59291) 

This adds some optional extra configuration to Parameter:

* custom serialization (to handle analyzers)
* deprecated parameter names
* parameter validation
This commit is contained in:
Alan Woodward 2020-07-13 12:26:53 +01:00 committed by Alan Woodward
parent 08b54feaaf
commit 19ba6c39d2
8 changed files with 357 additions and 283 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java
View File

@ -144,7 +144,7 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase {
assertThat(mappingMetadata, not(nullValue()));
Map<String, Object> mappingSource = mappingMetadata.sourceAsMap();
Map<String, Object> aField = ((Map<String, Object>) XContentMapValues.extractValue("properties.a", mappingSource));
assertThat(aField.size(), equalTo(6));
assertThat(aField.size(), equalTo(2));
assertThat(aField.get("type").toString(), equalTo("completion"));
assertThat(aField.get("fields"), notNullValue());

3
server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java
View File

@ -59,8 +59,7 @@ public class BinaryFieldMapper extends ParametrizedFieldMapper {
private final Parameter<Boolean> stored = Parameter.boolParam("store", false, m -> toType(m).stored, false);
private final Parameter<Boolean> hasDocValues = Parameter.boolParam("doc_values", false, m -> toType(m).hasDocValues, false);
private final Parameter<Map<String, String>> meta
= new Parameter<>("meta", true, Collections.emptyMap(), TypeParsers::parseMeta, m -> m.fieldType().meta());
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
public Builder(String name) {
this(name, false);

5
server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
View File

@ -80,11 +80,10 @@ public class BooleanFieldMapper extends ParametrizedFieldMapper {
private final Parameter<Boolean> stored = Parameter.boolParam("store", false, m -> toType(m).stored, false);
private final Parameter<Boolean> nullValue
= new Parameter<>("null_value", false, null, (n, o) -> XContentMapValues.nodeBooleanValue(o), m -> toType(m).nullValue);
= new Parameter<>("null_value", false, null, (n, c, o) -> XContentMapValues.nodeBooleanValue(o), m -> toType(m).nullValue);
private final Parameter<Float> boost = Parameter.floatParam("boost", true, m -> m.fieldType().boost(), 1.0f);
private final Parameter<Map<String, String>> meta
= new Parameter<>("meta", true, Collections.emptyMap(), TypeParsers::parseMeta, m -> m.fieldType().meta());
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
public Builder(String name) {
super(name);

335
server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
View File

@ -34,14 +34,11 @@ import org.apache.lucene.search.suggest.document.RegexCompletionQuery;
import org.apache.lucene.search.suggest.document.SuggestField;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.NumberType;
import org.elasticsearch.common.xcontent.XContentParser.Token;
@ -53,16 +50,14 @@ import org.elasticsearch.search.suggest.completion.context.ContextMapping;
import org.elasticsearch.search.suggest.completion.context.ContextMappings;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.elasticsearch.index.mapper.TypeParsers.parseMultiField;
/**
* Mapper for completion field. The field values are indexed as a weighted FST for
* fast auto-completion/search-as-you-type functionality.<br>
@ -84,7 +79,7 @@ import static org.elasticsearch.index.mapper.TypeParsers.parseMultiField;
* This field can also be extended to add search criteria to suggestions
* for query-time filtering and boosting (see {@link ContextMappings}
*/
public class CompletionFieldMapper extends FieldMapper {
public class CompletionFieldMapper extends ParametrizedFieldMapper {
public static final String CONTENT_TYPE = "completion";
/**
@ -92,6 +87,11 @@ public class CompletionFieldMapper extends FieldMapper {
*/
static final int COMPLETION_CONTEXTS_LIMIT = 10;
@Override
public ParametrizedFieldMapper.Builder getMergeBuilder() {
return new Builder(simpleName(), defaultAnalyzer).init(this);
}
public static class Defaults {
public static final FieldType FIELD_TYPE = new FieldType();
static {
@ -108,20 +108,97 @@ public class CompletionFieldMapper extends FieldMapper {
}
public static class Fields {
// Mapping field names
public static final ParseField ANALYZER = new ParseField("analyzer");
public static final ParseField SEARCH_ANALYZER = new ParseField("search_analyzer");
public static final ParseField PRESERVE_SEPARATORS = new ParseField("preserve_separators");
public static final ParseField PRESERVE_POSITION_INCREMENTS = new ParseField("preserve_position_increments");
public static final ParseField TYPE = new ParseField("type");
public static final ParseField CONTEXTS = new ParseField("contexts");
public static final ParseField MAX_INPUT_LENGTH = new ParseField("max_input_length", "max_input_len");
// Content field names
public static final String CONTENT_FIELD_NAME_INPUT = "input";
public static final String CONTENT_FIELD_NAME_WEIGHT = "weight";
public static final String CONTENT_FIELD_NAME_CONTEXTS = "contexts";
}
private static CompletionFieldMapper toType(FieldMapper in) {
return (CompletionFieldMapper) in;
}
/**
* Builder for {@link CompletionFieldMapper}
*/
public static class Builder extends ParametrizedFieldMapper.Builder {
private final Parameter<NamedAnalyzer> analyzer;
private final Parameter<NamedAnalyzer> searchAnalyzer;
private final Parameter<Boolean> preserveSeparators = Parameter.boolParam("preserve_separators", false,
m -> toType(m).preserveSeparators, Defaults.DEFAULT_PRESERVE_SEPARATORS);
private final Parameter<Boolean> preservePosInc = Parameter.boolParam("preserve_position_increments", false,
m -> toType(m).preservePosInc, Defaults.DEFAULT_POSITION_INCREMENTS);
private final Parameter<ContextMappings> contexts = new Parameter<>("contexts", false, null,
(n, c, o) -> ContextMappings.load(o, c.indexVersionCreated()), m -> toType(m).contexts)
.setSerializer((b, n, c) -> {
if (c == null) {
return;
}
b.startArray(n);
c.toXContent(b, ToXContent.EMPTY_PARAMS);
b.endArray();
});
private final Parameter<Integer> maxInputLength = Parameter.intParam("max_input_length", true,
m -> toType(m).maxInputLength, Defaults.DEFAULT_MAX_INPUT_LENGTH)
.addDeprecatedName("max_input_len")
.setValidator(Builder::validateInputLength);
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
private final NamedAnalyzer defaultAnalyzer;
private static final DeprecationLogger deprecationLogger
= new DeprecationLogger(LogManager.getLogger(CompletionFieldMapper.class));
/**
* @param name of the completion field to build
*/
public Builder(String name, NamedAnalyzer defaultAnalyzer) {
super(name);
this.defaultAnalyzer = defaultAnalyzer;
this.analyzer = Parameter.analyzerParam("analyzer", false, m -> toType(m).analyzer, defaultAnalyzer);
this.searchAnalyzer = Parameter.analyzerParam("search_analyzer", true, m -> toType(m).searchAnalyzer, defaultAnalyzer);
}
private static void validateInputLength(int maxInputLength) {
if (maxInputLength <= 0) {
throw new IllegalArgumentException("[max_input_length] must be > 0 but was [" + maxInputLength + "]");
}
}
@Override
protected List<Parameter<?>> getParameters() {
return Arrays.asList(analyzer, searchAnalyzer, preserveSeparators, preservePosInc, contexts, maxInputLength, meta);
}
@Override
public CompletionFieldMapper build(BuilderContext context) {
checkCompletionContextsLimit(context);
NamedAnalyzer completionAnalyzer = new NamedAnalyzer(this.searchAnalyzer.getValue().name(), AnalyzerScope.INDEX,
new CompletionAnalyzer(this.searchAnalyzer.getValue(), preserveSeparators.getValue(), preservePosInc.getValue()));
CompletionFieldType ft
= new CompletionFieldType(buildFullName(context), completionAnalyzer, meta.getValue());
ft.setContextMappings(contexts.getValue());
ft.setPreservePositionIncrements(preservePosInc.getValue());
ft.setPreserveSep(preserveSeparators.getValue());
ft.setIndexAnalyzer(analyzer.getValue());
return new CompletionFieldMapper(name, ft, defaultAnalyzer,
multiFieldsBuilder.build(this, context), copyTo.build(), this);
}
private void checkCompletionContextsLimit(BuilderContext context) {
if (this.contexts.getValue() != null && this.contexts.getValue().size() > COMPLETION_CONTEXTS_LIMIT) {
deprecationLogger.deprecatedAndMaybeLog("excessive_completion_contexts",
"You have defined more than [" + COMPLETION_CONTEXTS_LIMIT + "] completion contexts" +
" in the mapping for index [" + context.indexSettings().get(IndexMetadata.SETTING_INDEX_PROVIDED_NAME) + "]. " +
"The maximum allowed number of completion contexts in a mapping will be limited to " +
"[" + COMPLETION_CONTEXTS_LIMIT + "] starting in version [8.0].");
}
}
}
public static final Set<String> ALLOWED_CONTENT_FIELD_NAMES = Sets.newHashSet(Fields.CONTENT_FIELD_NAME_INPUT,
Fields.CONTENT_FIELD_NAME_WEIGHT, Fields.CONTENT_FIELD_NAME_CONTEXTS);
@ -130,60 +207,11 @@ public class CompletionFieldMapper extends FieldMapper {
@Override
public Mapper.Builder<?> parse(String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
CompletionFieldMapper.Builder builder = new CompletionFieldMapper.Builder(name);
NamedAnalyzer indexAnalyzer = null;
NamedAnalyzer searchAnalyzer = null;
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
Map.Entry<String, Object> entry = iterator.next();
String fieldName = entry.getKey();
Object fieldNode = entry.getValue();
if (fieldName.equals("type")) {
continue;
}
if (Fields.ANALYZER.match(fieldName, LoggingDeprecationHandler.INSTANCE)) {
indexAnalyzer = getNamedAnalyzer(parserContext, fieldNode.toString());
iterator.remove();
} else if (Fields.SEARCH_ANALYZER.match(fieldName, LoggingDeprecationHandler.INSTANCE)) {
searchAnalyzer = getNamedAnalyzer(parserContext, fieldNode.toString());
iterator.remove();
} else if (Fields.PRESERVE_SEPARATORS.match(fieldName, LoggingDeprecationHandler.INSTANCE)) {
builder.preserveSeparators(Boolean.parseBoolean(fieldNode.toString()));
iterator.remove();
} else if (Fields.PRESERVE_POSITION_INCREMENTS.match(fieldName, LoggingDeprecationHandler.INSTANCE)) {
builder.preservePositionIncrements(Boolean.parseBoolean(fieldNode.toString()));
iterator.remove();
} else if (Fields.MAX_INPUT_LENGTH.match(fieldName, LoggingDeprecationHandler.INSTANCE)) {
builder.maxInputLength(Integer.parseInt(fieldNode.toString()));
iterator.remove();
} else if (Fields.CONTEXTS.match(fieldName, LoggingDeprecationHandler.INSTANCE)) {
builder.contextMappings(ContextMappings.load(fieldNode, parserContext.indexVersionCreated()));
iterator.remove();
} else if (parseMultiField(builder::addMultiField, name, parserContext, fieldName, fieldNode)) {
iterator.remove();
}
}
if (indexAnalyzer == null) {
if (searchAnalyzer != null) {
throw new MapperParsingException("analyzer on completion field [" + name + "] must be set when search_analyzer is set");
}
indexAnalyzer = searchAnalyzer = parserContext.getIndexAnalyzers().get("simple");
} else if (searchAnalyzer == null) {
searchAnalyzer = indexAnalyzer;
}
builder.indexAnalyzer(indexAnalyzer);
builder.searchAnalyzer(searchAnalyzer);
CompletionFieldMapper.Builder builder
= new CompletionFieldMapper.Builder(name, parserContext.getIndexAnalyzers().get("simple"));
builder.parse(name, parserContext, node);
return builder;
}
private NamedAnalyzer getNamedAnalyzer(ParserContext parserContext, String name) {
NamedAnalyzer analyzer = parserContext.getIndexAnalyzers().get(name);
if (analyzer == null) {
throw new IllegalArgumentException("Can't find default or mapped analyzer with name [" + name + "]");
}
return analyzer;
}
}
public static final class CompletionFieldType extends TermBasedFieldType {
@ -194,17 +222,9 @@ public class CompletionFieldMapper extends FieldMapper {
private boolean preservePositionIncrements = Defaults.DEFAULT_POSITION_INCREMENTS;
private ContextMappings contextMappings = null;
public CompletionFieldType(String name, FieldType luceneFieldType,
NamedAnalyzer searchAnalyzer, NamedAnalyzer searchQuoteAnalyzer, Map<String, String> meta) {
public CompletionFieldType(String name, NamedAnalyzer searchAnalyzer, Map<String, String> meta) {
super(name, true, false,
new TextSearchInfo(luceneFieldType, null, searchAnalyzer, searchQuoteAnalyzer), meta);
}
public CompletionFieldType(String name) {
this(name, Defaults.FIELD_TYPE,
new NamedAnalyzer("completion", AnalyzerScope.INDEX, new CompletionAnalyzer(Lucene.STANDARD_ANALYZER)),
new NamedAnalyzer("completion", AnalyzerScope.INDEX, new CompletionAnalyzer(Lucene.STANDARD_ANALYZER)),
Collections.emptyMap());
new TextSearchInfo(Defaults.FIELD_TYPE, null, searchAnalyzer, searchAnalyzer), meta);
}
public void setPreserveSep(boolean preserveSep) {
@ -245,14 +265,6 @@ public class CompletionFieldMapper extends FieldMapper {
return contextMappings;
}
public boolean preserveSep() {
return preserveSep;
}
public boolean preservePositionIncrements() {
return preservePositionIncrements;
}
/**
* @return postings format to use for this field-type
*/
@ -302,100 +314,24 @@ public class CompletionFieldMapper extends FieldMapper {
}
/**
* Builder for {@link CompletionFieldMapper}
*/
public static class Builder extends FieldMapper.Builder<Builder> {
private final int maxInputLength;
private final boolean preserveSeparators;
private final boolean preservePosInc;
private final NamedAnalyzer defaultAnalyzer;
private final NamedAnalyzer analyzer;
private final NamedAnalyzer searchAnalyzer;
private final ContextMappings contexts;
private int maxInputLength = Defaults.DEFAULT_MAX_INPUT_LENGTH;
private ContextMappings contextMappings = null;
private boolean preserveSeparators = Defaults.DEFAULT_PRESERVE_SEPARATORS;
private boolean preservePositionIncrements = Defaults.DEFAULT_POSITION_INCREMENTS;
private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(Builder.class));
/**
* @param name of the completion field to build
*/
public Builder(String name) {
super(name, Defaults.FIELD_TYPE);
builder = this;
}
/**
* @param maxInputLength maximum expected prefix length
* NOTE: prefixes longer than this will
* be truncated
*/
public Builder maxInputLength(int maxInputLength) {
if (maxInputLength <= 0) {
throw new IllegalArgumentException(Fields.MAX_INPUT_LENGTH.getPreferredName()
+ " must be > 0 but was [" + maxInputLength + "]");
}
this.maxInputLength = maxInputLength;
return this;
}
/**
* Add context mapping to this field
* @param contextMappings see {@link ContextMappings#load(Object, Version)}
*/
public Builder contextMappings(ContextMappings contextMappings) {
this.contextMappings = contextMappings;
return this;
}
public Builder preserveSeparators(boolean preserveSeparators) {
this.preserveSeparators = preserveSeparators;
return this;
}
public Builder preservePositionIncrements(boolean preservePositionIncrements) {
this.preservePositionIncrements = preservePositionIncrements;
return this;
}
@Override
public CompletionFieldMapper build(BuilderContext context) {
checkCompletionContextsLimit(context);
NamedAnalyzer searchAnalyzer = new NamedAnalyzer(this.searchAnalyzer.name(), AnalyzerScope.INDEX,
new CompletionAnalyzer(this.searchAnalyzer, preserveSeparators, preservePositionIncrements));
CompletionFieldType ft
= new CompletionFieldType(buildFullName(context), this.fieldType, searchAnalyzer, searchAnalyzer, meta);
ft.setContextMappings(contextMappings);
ft.setPreservePositionIncrements(preservePositionIncrements);
ft.setPreserveSep(preserveSeparators);
ft.setIndexAnalyzer(indexAnalyzer);
return new CompletionFieldMapper(name, this.fieldType, ft,
multiFieldsBuilder.build(this, context), copyTo, maxInputLength);
}
private void checkCompletionContextsLimit(BuilderContext context) {
if (this.contextMappings != null && this.contextMappings.size() > COMPLETION_CONTEXTS_LIMIT) {
deprecationLogger.deprecatedAndMaybeLog("excessive_completion_contexts",
"You have defined more than [" + COMPLETION_CONTEXTS_LIMIT + "] completion contexts" +
" in the mapping for index [" + context.indexSettings().get(IndexMetadata.SETTING_INDEX_PROVIDED_NAME) + "]. " +
"The maximum allowed number of completion contexts in a mapping will be limited to " +
"[" + COMPLETION_CONTEXTS_LIMIT + "] starting in version [8.0].");
}
}
@Override
public Builder index(boolean index) {
if (index == false) {
throw new MapperParsingException("Completion field type must be indexed");
}
return builder;
}
}
private int maxInputLength;
public CompletionFieldMapper(String simpleName, FieldType fieldType, MappedFieldType mappedFieldType,
MultiFields multiFields, CopyTo copyTo, int maxInputLength) {
super(simpleName, fieldType, mappedFieldType, multiFields, copyTo);
this.maxInputLength = maxInputLength;
public CompletionFieldMapper(String simpleName, MappedFieldType mappedFieldType, NamedAnalyzer defaultAnalyzer,
MultiFields multiFields, CopyTo copyTo, Builder builder) {
super(simpleName, mappedFieldType, multiFields, copyTo);
this.defaultAnalyzer = defaultAnalyzer;
this.maxInputLength = builder.maxInputLength.getValue();
this.preserveSeparators = builder.preserveSeparators.getValue();
this.preservePosInc = builder.preservePosInc.getValue();
this.analyzer = builder.analyzer.getValue();
this.searchAnalyzer = builder.searchAnalyzer.getValue();
this.contexts = builder.contexts.getValue();
}
@Override
@ -601,28 +537,6 @@ public class CompletionFieldMapper extends FieldMapper {
}
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(simpleName())
.field(Fields.TYPE.getPreferredName(), CONTENT_TYPE);
builder.field(Fields.ANALYZER.getPreferredName(), fieldType().indexAnalyzer().name());
if (fieldType().indexAnalyzer().name().equals(fieldType().getTextSearchInfo().getSearchAnalyzer().name()) == false) {
builder.field(Fields.SEARCH_ANALYZER.getPreferredName(), fieldType().getTextSearchInfo().getSearchAnalyzer().name());
}
builder.field(Fields.PRESERVE_SEPARATORS.getPreferredName(), fieldType().preserveSep());
builder.field(Fields.PRESERVE_POSITION_INCREMENTS.getPreferredName(), fieldType().preservePositionIncrements());
builder.field(Fields.MAX_INPUT_LENGTH.getPreferredName(), this.maxInputLength);
if (fieldType().hasContextMappings()) {
builder.startArray(Fields.CONTEXTS.getPreferredName());
fieldType().getContextMappings().toXContent(builder, params);
builder.endArray();
}
multiFields.toXContent(builder, params);
return builder.endObject();
}
@Override
protected void parseCreateField(ParseContext context) throws IOException {
// no-op
@ -633,23 +547,4 @@ public class CompletionFieldMapper extends FieldMapper {
return CONTENT_TYPE;
}
@Override
protected void mergeOptions(FieldMapper other, List<String> conflicts) {
CompletionFieldType c = (CompletionFieldType)other.fieldType();
if (fieldType().preservePositionIncrements != c.preservePositionIncrements) {
conflicts.add("mapper [" + name() + "] has different [preserve_position_increments] values");
}
if (fieldType().preserveSep != c.preserveSep) {
conflicts.add("mapper [" + name() + "] has different [preserve_separators] values");
}
if (fieldType().hasContextMappings() != c.hasContextMappings()) {
conflicts.add("mapper [" + name() + "] has different [context_mappings] values");
} else if (fieldType().hasContextMappings() && fieldType().contextMappings.equals(c.contextMappings) == false) {
conflicts.add("mapper [" + name() + "] has different [context_mappings] values");
}
this.maxInputLength = ((CompletionFieldMapper)other).maxInputLength;
}
}

136
server/src/main/java/org/elasticsearch/index/mapper/ParametrizedFieldMapper.java
View File

@ -21,23 +21,27 @@ package org.elasticsearch.index.mapper;
import org.apache.logging.log4j.LogManager;
import org.apache.lucene.document.FieldType;
import org.elasticsearch.common.TriFunction;
import org.elasticsearch.Version;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.Mapper.TypeParser.ParserContext;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Consumer;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.Function;
/**
@ -113,6 +117,13 @@ public abstract class ParametrizedFieldMapper extends FieldMapper {
return builder.endObject();
}
/**
* Serializes a parameter
*/
protected interface Serializer<T> {
void serialize(XContentBuilder builder, String name, T value) throws IOException;
}
/**
* A configurable parameter for a field mapper
* @param <T> the type of the value the parameter holds
@ -120,11 +131,14 @@ public abstract class ParametrizedFieldMapper extends FieldMapper {
public static final class Parameter<T> {
public final String name;
private final List<String> deprecatedNames = new ArrayList<>();
private final T defaultValue;
private final BiFunction<String, Object, T> parser;
private final TriFunction<String, ParserContext, Object, T> parser;
private final Function<FieldMapper, T> initializer;
private final boolean updateable;
private boolean acceptsNull = false;
private Consumer<T> validator = null;
private Serializer<T> serializer = XContentBuilder::field;
private T value;
/**
@ -136,7 +150,7 @@ public abstract class ParametrizedFieldMapper extends FieldMapper {
* @param initializer a function that reads a parameter value from an existing mapper
*/
public Parameter(String name, boolean updateable, T defaultValue,
BiFunction<String, Object, T> parser, Function<FieldMapper, T> initializer) {
TriFunction<String, ParserContext, Object, T> parser, Function<FieldMapper, T> initializer) {
this.name = name;
this.defaultValue = defaultValue;
this.value = defaultValue;
@ -167,12 +181,45 @@ public abstract class ParametrizedFieldMapper extends FieldMapper {
return this;
}
private void init(FieldMapper toInit) {
this.value = initializer.apply(toInit);
/**
* Adds a deprecated parameter name.
*
* If this parameter name is encountered during parsing, a deprecation warning will
* be emitted. The parameter will be serialized with its main name.
*/
public Parameter<T> addDeprecatedName(String deprecatedName) {
this.deprecatedNames.add(deprecatedName);
return this;
}
private void parse(String field, Object in) {
this.value = parser.apply(field, in);
/**
* Adds validation to a parameter, called after parsing and merging
*/
public Parameter<T> setValidator(Consumer<T> validator) {
this.validator = validator;
return this;
}
/**
* Configure a custom serializer for this parameter
*/
public Parameter<T> setSerializer(Serializer<T> serializer) {
this.serializer = serializer;
return this;
}
private void validate() {
if (validator != null) {
validator.accept(value);
}
}
private void init(FieldMapper toInit) {
setValue(initializer.apply(toInit));
}
private void parse(String field, ParserContext context, Object in) {
setValue(parser.apply(field, context, in));
}
private void merge(FieldMapper toMerge, Conflicts conflicts) {
@ -180,13 +227,13 @@ public abstract class ParametrizedFieldMapper extends FieldMapper {
if (updateable == false && Objects.equals(this.value, value) == false) {
conflicts.addConflict(name, this.value.toString(), value.toString());
} else {
this.value = value;
setValue(value);
}
}
private void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException {
if (includeDefaults || (Objects.equals(defaultValue, value) == false)) {
builder.field(name, value);
serializer.serialize(builder, name, value);
}
}
@ -199,7 +246,7 @@ public abstract class ParametrizedFieldMapper extends FieldMapper {
*/
public static Parameter<Boolean> boolParam(String name, boolean updateable,
Function<FieldMapper, Boolean> initializer, boolean defaultValue) {
return new Parameter<>(name, updateable, defaultValue, (n, o) -> XContentMapValues.nodeBooleanValue(o), initializer);
return new Parameter<>(name, updateable, defaultValue, (n, c, o) -> XContentMapValues.nodeBooleanValue(o), initializer);
}
/**
@ -211,7 +258,19 @@ public abstract class ParametrizedFieldMapper extends FieldMapper {
*/
public static Parameter<Float> floatParam(String name, boolean updateable,
Function<FieldMapper, Float> initializer, float defaultValue) {
return new Parameter<>(name, updateable, defaultValue, (n, o) -> XContentMapValues.nodeFloatValue(o), initializer);
return new Parameter<>(name, updateable, defaultValue, (n, c, o) -> XContentMapValues.nodeFloatValue(o), initializer);
}
/**
* Defines a parameter that takes an integer value
* @param name the parameter name
* @param updateable whether the parameter can be changed by a mapping update
* @param initializer a function that reads the parameter value from an existing mapper
* @param defaultValue the default value, to be used if the parameter is undefined in a mapping
*/
public static Parameter<Integer> intParam(String name, boolean updateable,
Function<FieldMapper, Integer> initializer, int defaultValue) {
return new Parameter<>(name, updateable, defaultValue, (n, c, o) -> XContentMapValues.nodeIntegerValue(o), initializer);
}
/**
@ -224,8 +283,37 @@ public abstract class ParametrizedFieldMapper extends FieldMapper {
public static Parameter<String> stringParam(String name, boolean updateable,
Function<FieldMapper, String> initializer, String defaultValue) {
return new Parameter<>(name, updateable, defaultValue,
(n, o) -> XContentMapValues.nodeStringValue(o), initializer);
(n, c, o) -> XContentMapValues.nodeStringValue(o), initializer);
}
/**
* Defines a parameter that takes an analyzer name
* @param name the parameter name
* @param updateable whether the parameter can be changed by a mapping update
* @param initializer a function that reads the parameter value from an existing mapper
* @param defaultAnalyzer the default value, to be used if the parameter is undefined in a mapping
*/
public static Parameter<NamedAnalyzer> analyzerParam(String name, boolean updateable,
Function<FieldMapper, NamedAnalyzer> initializer,
NamedAnalyzer defaultAnalyzer) {
return new Parameter<>(name, updateable, defaultAnalyzer, (n, c, o) -> {
String analyzerName = o.toString();
NamedAnalyzer a = c.getIndexAnalyzers().get(analyzerName);
if (a == null) {
throw new IllegalArgumentException("analyzer [" + analyzerName + "] has not been configured in mappings");
}
return a;
}, initializer).setSerializer((b, n, v) -> b.field(n, v.name()));
}
/**
* Declares a metadata parameter
*/
public static Parameter<Map<String, String>> metaParam() {
return new Parameter<>("meta", true, Collections.emptyMap(),
(n, c, o) -> TypeParsers.parseMeta(n, o), m -> m.fieldType().meta());
}
}
private static final class Conflicts {
@ -288,6 +376,13 @@ public abstract class ParametrizedFieldMapper extends FieldMapper {
multiFieldsBuilder.update(newSubField, parentPath(newSubField.name()));
}
this.copyTo.reset(in.copyTo);
validate();
}
private void validate() {
for (Parameter<?> param : getParameters()) {
param.validate();
}
}
/**
@ -317,10 +412,14 @@ public abstract class ParametrizedFieldMapper extends FieldMapper {
* @param parserContext the parser context
* @param fieldNode the root node of the map of mappings for this field
*/
public final void parse(String name, TypeParser.ParserContext parserContext, Map<String, Object> fieldNode) {
public final void parse(String name, ParserContext parserContext, Map<String, Object> fieldNode) {
Map<String, Parameter<?>> paramsMap = new HashMap<>();
Map<String, Parameter<?>> deprecatedParamsMap = new HashMap<>();
for (Parameter<?> param : getParameters()) {
paramsMap.put(param.name, param);
for (String deprecatedName : param.deprecatedNames) {
deprecatedParamsMap.put(deprecatedName, param);
}
}
String type = (String) fieldNode.remove("type");
for (Iterator<Map.Entry<String, Object>> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) {
@ -337,7 +436,13 @@ public abstract class ParametrizedFieldMapper extends FieldMapper {
iterator.remove();
continue;
}
Parameter<?> parameter = paramsMap.get(propName);
Parameter<?> parameter = deprecatedParamsMap.get(propName);
if (parameter != null) {
deprecationLogger.deprecatedAndMaybeLog(propName, "Parameter [{}] on mapper [{}] is deprecated, use [{}]",
propName, name, parameter.name);
} else {
parameter = paramsMap.get(propName);
}
if (parameter == null) {
if (isDeprecatedParameter(propName, parserContext.indexVersionCreated())) {
deprecationLogger.deprecatedAndMaybeLog(propName,
@ -352,9 +457,10 @@ public abstract class ParametrizedFieldMapper extends FieldMapper {
throw new MapperParsingException("[" + propName + "] on mapper [" + name
+ "] of type [" + type + "] must not have a [null] value");
}
parameter.parse(name, propNode);
parameter.parse(name, parserContext, propNode);
iterator.remove();
}
validate();
}
// These parameters were previously *always* parsed by TypeParsers#parseField(), even if they

37
server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
View File

@ -18,7 +18,6 @@
*/
package org.elasticsearch.index.mapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.search.Query;
@ -43,20 +42,16 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.search.suggest.completion.context.ContextBuilder;
import org.elasticsearch.search.suggest.completion.context.ContextMappings;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.hamcrest.FeatureMatcher;
import org.hamcrest.Matcher;
import org.hamcrest.Matchers;
import org.hamcrest.core.CombinableMatcher;
import org.junit.Before;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@ -68,31 +63,7 @@ import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
public class CompletionFieldMapperTests extends FieldMapperTestCase<CompletionFieldMapper.Builder> {
@Before
public void addModifiers() {
addBooleanModifier("preserve_separators", false, CompletionFieldMapper.Builder::preserveSeparators);
addBooleanModifier("preserve_position_increments", false, CompletionFieldMapper.Builder::preservePositionIncrements);
addModifier("context_mappings", false, (a, b) -> {
ContextMappings contextMappings = new ContextMappings(Arrays.asList(ContextBuilder.category("foo").build(),
ContextBuilder.geo("geo").build()));
a.contextMappings(contextMappings);
});
}
@Override
protected Set<String> unsupportedProperties() {
return org.elasticsearch.common.collect.Set.of("doc_values", "index");
}
@Override
protected CompletionFieldMapper.Builder newBuilder() {
CompletionFieldMapper.Builder builder = new CompletionFieldMapper.Builder("completion");
builder.indexAnalyzer(new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer()));
builder.searchAnalyzer(new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer()));
return builder;
}
public class CompletionFieldMapperTests extends ESSingleNodeTestCase {
public void testDefaultConfiguration() throws IOException {
String mapping = Strings.toString(jsonBuilder().startObject().startObject("type1")
@ -176,7 +147,7 @@ public class CompletionFieldMapperTests extends FieldMapperTestCase<CompletionFi
assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class));
XContentBuilder builder = jsonBuilder().startObject();
fieldMapper.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject();
fieldMapper.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("include_defaults", "true"))).endObject();
builder.close();
Map<String, Object> serializedMap = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder)).map();
Map<String, Object> configMap = (Map<String, Object>) serializedMap.get("completion");

116
server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java
View File

@ -19,15 +19,20 @@
package org.elasticsearch.index.mapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.elasticsearch.Version;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.ParametrizedFieldMapper.Parameter;
import org.elasticsearch.plugins.MapperPlugin;
import org.elasticsearch.plugins.Plugin;
@ -37,10 +42,15 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import static org.hamcrest.Matchers.instanceOf;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class ParametrizedMapperTests extends ESSingleNodeTestCase {
public static class TestPlugin extends Plugin implements MapperPlugin {
@ -55,6 +65,27 @@ public class ParametrizedMapperTests extends ESSingleNodeTestCase {
return Collections.singletonList(TestPlugin.class);
}
private static class StringWrapper {
final String name;
private StringWrapper(String name) {
this.name = name;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
StringWrapper that = (StringWrapper) o;
return Objects.equals(name, that.name);
}
@Override
public int hashCode() {
return Objects.hash(name);
}
}
private static TestMapper toType(Mapper in) {
return (TestMapper) in;
}
@ -64,9 +95,25 @@ public class ParametrizedMapperTests extends ESSingleNodeTestCase {
final Parameter<Boolean> fixed
= Parameter.boolParam("fixed", false, m -> toType(m).fixed, true);
final Parameter<Boolean> fixed2
= Parameter.boolParam("fixed2", false, m -> toType(m).fixed2, false);
= Parameter.boolParam("fixed2", false, m -> toType(m).fixed2, false)
.addDeprecatedName("fixed2_old");
final Parameter<String> variable
= Parameter.stringParam("variable", true, m -> toType(m).variable, "default").acceptsNull();
final Parameter<StringWrapper> wrapper
= new Parameter<>("wrapper", true, new StringWrapper("default"),
(n, c, o) -> {
if (o == null) return null;
return new StringWrapper(o.toString());
},
m -> toType(m).wrapper).setSerializer((b, n, v) -> b.field(n, v.name));
final Parameter<Integer> intValue = Parameter.intParam("int_value", true, m -> toType(m).intValue, 5)
.setValidator(n -> {
if (n > 50) {
throw new IllegalArgumentException("Value of [n] cannot be greater than 50");
}
});
final Parameter<NamedAnalyzer> analyzer
= Parameter.analyzerParam("analyzer", true, m -> toType(m).analyzer, Lucene.KEYWORD_ANALYZER);
final Parameter<Boolean> index = Parameter.boolParam("index", false, m -> toType(m).index, true);
protected Builder(String name) {
@ -75,7 +122,7 @@ public class ParametrizedMapperTests extends ESSingleNodeTestCase {
@Override
protected List<Parameter<?>> getParameters() {
return Arrays.asList(fixed, fixed2, variable, index);
return Arrays.asList(fixed, fixed2, variable, index, wrapper, intValue, analyzer);
}
@Override
@ -100,6 +147,9 @@ public class ParametrizedMapperTests extends ESSingleNodeTestCase {
private final boolean fixed;
private final boolean fixed2;
private final String variable;
private final StringWrapper wrapper;
private final int intValue;
private final NamedAnalyzer analyzer;
private final boolean index;
protected TestMapper(String simpleName, String fullName, MultiFields multiFields, CopyTo copyTo,
@ -108,6 +158,9 @@ public class ParametrizedMapperTests extends ESSingleNodeTestCase {
this.fixed = builder.fixed.getValue();
this.fixed2 = builder.fixed2.getValue();
this.variable = builder.variable.getValue();
this.wrapper = builder.wrapper.getValue();
this.intValue = builder.intValue.getValue();
this.analyzer = builder.analyzer.getValue();
this.index = builder.index.getValue();
}
@ -128,7 +181,14 @@ public class ParametrizedMapperTests extends ESSingleNodeTestCase {
}
private static TestMapper fromMapping(String mapping, Version version) {
Mapper.TypeParser.ParserContext pc = new Mapper.TypeParser.ParserContext(s -> null, null, s -> {
MapperService mapperService = mock(MapperService.class);
Map<String, NamedAnalyzer> analyzers = new HashMap<>();
analyzers.put("_standard", Lucene.STANDARD_ANALYZER);
analyzers.put("_keyword", Lucene.KEYWORD_ANALYZER);
analyzers.put("default", new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer()));
IndexAnalyzers indexAnalyzers = new IndexAnalyzers(analyzers, Collections.emptyMap(), Collections.emptyMap());
when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers);
Mapper.TypeParser.ParserContext pc = new Mapper.TypeParser.ParserContext(s -> null, mapperService, s -> {
if (Objects.equals("keyword", s)) {
return new KeywordFieldMapper.TypeParser();
}
@ -162,7 +222,8 @@ public class ParametrizedMapperTests extends ESSingleNodeTestCase {
mapper.toXContent(builder, params);
builder.endObject();
assertEquals("{\"field\":{\"type\":\"test_mapper\",\"fixed\":true," +
"\"fixed2\":false,\"variable\":\"default\",\"index\":true}}",
"\"fixed2\":false,\"variable\":\"default\",\"index\":true," +
"\"wrapper\":\"default\",\"int_value\":5,\"analyzer\":\"_keyword\"}}",
Strings.toString(builder));
}
@ -255,6 +316,53 @@ public class ParametrizedMapperTests extends ESSingleNodeTestCase {
assertEquals(mapping, Strings.toString(indexService.mapperService().documentMapper()));
}
// test custom serializer
public void testCustomSerialization() {
String mapping = "{\"type\":\"test_mapper\",\"wrapper\":\"wrapped value\"}";
TestMapper mapper = fromMapping(mapping);
assertEquals("wrapped value", mapper.wrapper.name);
assertEquals("{\"field\":" + mapping + "}", Strings.toString(mapper));
}
// test validator
public void testParameterValidation() {
String mapping = "{\"type\":\"test_mapper\",\"int_value\":10}";
TestMapper mapper = fromMapping(mapping);
assertEquals(10, mapper.intValue);
assertEquals("{\"field\":" + mapping + "}", Strings.toString(mapper));
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> fromMapping("{\"type\":\"test_mapper\",\"int_value\":60}"));
assertEquals("Value of [n] cannot be greater than 50", e.getMessage());
}
// test deprecations
public void testDeprecatedParameterName() {
String mapping = "{\"type\":\"test_mapper\",\"fixed2_old\":true}";
TestMapper mapper = fromMapping(mapping);
assertTrue(mapper.fixed2);
assertWarnings("Parameter [fixed2_old] on mapper [field] is deprecated, use [fixed2]");
assertEquals("{\"field\":{\"type\":\"test_mapper\",\"fixed2\":true}}", Strings.toString(mapper));
}
public void testAnalyzers() {
String mapping = "{\"type\":\"test_mapper\",\"analyzer\":\"_standard\"}";
TestMapper mapper = fromMapping(mapping);
assertEquals(mapper.analyzer, Lucene.STANDARD_ANALYZER);
assertEquals("{\"field\":" + mapping + "}", Strings.toString(mapper));
String withDef = "{\"type\":\"test_mapper\",\"analyzer\":\"default\"}";
mapper = fromMapping(withDef);
assertEquals(mapper.analyzer.name(), "default");
assertThat(mapper.analyzer.analyzer(), instanceOf(StandardAnalyzer.class));
assertEquals("{\"field\":" + withDef + "}", Strings.toString(mapper));
String badAnalyzer = "{\"type\":\"test_mapper\",\"analyzer\":\"wibble\"}";
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> fromMapping(badAnalyzer));
assertEquals("analyzer [wibble] has not been configured in mappings", e.getMessage());
}
public void testDeprecatedParameters() throws IOException {
// 'index' is declared explicitly, 'store' is not, but is one of the previously always-accepted params
String mapping = "{\"type\":\"test_mapper\",\"index\":false,\"store\":true}";

6
server/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java
View File

@ -25,7 +25,6 @@ import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.CompletionFieldMapper;
import org.elasticsearch.index.mapper.CompletionFieldMapper.CompletionFieldType;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase;
@ -169,14 +168,11 @@ public class CompletionSuggesterBuilderTests extends AbstractSuggestionBuilderTe
@Override
protected MappedFieldType mockFieldType(String fieldName, boolean analyzerSet) {
if (analyzerSet == false) {
CompletionFieldType completionFieldType = new CompletionFieldType(fieldName,
CompletionFieldMapper.Defaults.FIELD_TYPE, null, null, Collections.emptyMap());
CompletionFieldType completionFieldType = new CompletionFieldType(fieldName, null, Collections.emptyMap());
completionFieldType.setContextMappings(new ContextMappings(contextMappings));
return completionFieldType;
}
CompletionFieldType completionFieldType = new CompletionFieldType(fieldName,
CompletionFieldMapper.Defaults.FIELD_TYPE,
new NamedAnalyzer("fieldSearchAnalyzer", AnalyzerScope.INDEX, new SimpleAnalyzer()),
new NamedAnalyzer("fieldSearchAnalyzer", AnalyzerScope.INDEX, new SimpleAnalyzer()),
Collections.emptyMap());
completionFieldType.setContextMappings(new ContextMappings(contextMappings));