honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move MappedFieldType.similarity() to TextSearchInfo (#58439) 

Similarities only apply to a few text-based field types, but are currently set directly on
the base MappedFieldType class. This commit moves similarity information into
TextSearchInfo, and removes any mentions of it from MappedFieldType or FieldMapper.

It was previously possible to include a similarity parameter on a number of field types
that would then ignore this information. To make it obvious that this has no effect, setting
this parameter on non-text field types now issues a deprecation warning.
This commit is contained in:
Alan Woodward 2020-06-24 09:54:56 +01:00 committed by Alan Woodward
parent fcd8a432d9
commit d251a482e9
18 changed files with 166 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java
View File

@ -52,6 +52,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.similarity.SimilarityProvider;
import org.elasticsearch.index.similarity.SimilarityService;
import java.io.IOException;
import java.util.ArrayList;
@ -64,6 +66,7 @@ import java.util.Objects;
import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeIntegerValue;
import static org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType.hasGaps;
import static org.elasticsearch.index.mapper.TypeParsers.checkNull;
import static org.elasticsearch.index.mapper.TypeParsers.parseTextField;
/**
@ -113,30 +116,39 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper {
builder.indexAnalyzer(parserContext.getIndexAnalyzers().getDefaultIndexAnalyzer());
builder.searchAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchAnalyzer());
builder.searchQuoteAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer());
parseTextField(builder, name, node, parserContext);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
final Map.Entry<String, Object> entry = iterator.next();
final String fieldName = entry.getKey();
final Object fieldNode = entry.getValue();
checkNull(fieldName, fieldNode);
if (fieldName.equals("max_shingle_size")) {
builder.maxShingleSize(nodeIntegerValue(fieldNode));
iterator.remove();
} else if (fieldName.equals("similarity")) {
SimilarityProvider similarityProvider = TypeParsers.resolveSimilarity(parserContext, fieldName, fieldNode.toString());
builder.similarity(similarityProvider);
iterator.remove();
}
// TODO should we allow to configure the prefix field
}
parseTextField(builder, name, node, parserContext);
return builder;
}
}
public static class Builder extends FieldMapper.Builder<Builder> {
private int maxShingleSize = Defaults.MAX_SHINGLE_SIZE;
private SimilarityProvider similarity;
public Builder(String name) {
super(name, Defaults.FIELD_TYPE);
this.builder = this;
}
public void similarity(SimilarityProvider similarity) {
this.similarity = similarity;
}
public Builder maxShingleSize(int maxShingleSize) {
if (maxShingleSize < MAX_SHINGLE_SIZE_LOWER_BOUND || maxShingleSize > MAX_SHINGLE_SIZE_UPPER_BOUND) {
throw new MapperParsingException("[max_shingle_size] must be at least [" + MAX_SHINGLE_SIZE_LOWER_BOUND + "] and at most " +
@ -157,11 +169,10 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper {
@Override
public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) {
SearchAsYouTypeFieldType ft = new SearchAsYouTypeFieldType(buildFullName(context), fieldType, meta);
SearchAsYouTypeFieldType ft = new SearchAsYouTypeFieldType(buildFullName(context), fieldType, similarity, meta);
ft.setIndexAnalyzer(indexAnalyzer);
ft.setSearchAnalyzer(searchAnalyzer);
ft.setSearchQuoteAnalyzer(searchQuoteAnalyzer);
ft.setSimilarity(similarity);
// set up the prefix field
FieldType prefixft = new FieldType(fieldType);
@ -235,8 +246,8 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper {
PrefixFieldType prefixField;
ShingleFieldType[] shingleFields = new ShingleFieldType[0];
SearchAsYouTypeFieldType(String name, FieldType fieldType, Map<String, String> meta) {
super(name, fieldType.indexOptions() != IndexOptions.NONE, false, new TextSearchInfo(fieldType), meta);
SearchAsYouTypeFieldType(String name, FieldType fieldType, SimilarityProvider similarity, Map<String, String> meta) {
super(name, fieldType.indexOptions() != IndexOptions.NONE, false, new TextSearchInfo(fieldType, similarity), meta);
}
SearchAsYouTypeFieldType(SearchAsYouTypeFieldType other) {
@ -382,7 +393,7 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper {
final String parentField;
PrefixFieldType(String parentField, FieldType fieldType, int minChars, int maxChars) {
super(parentField + PREFIX_FIELD_SUFFIX, true, false, new TextSearchInfo(fieldType), Collections.emptyMap());
super(parentField + PREFIX_FIELD_SUFFIX, true, false, new TextSearchInfo(fieldType, null), Collections.emptyMap());
this.minChars = minChars;
this.maxChars = maxChars;
this.parentField = parentField;
@ -535,7 +546,7 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper {
PrefixFieldType prefixFieldType;
ShingleFieldType(String name, int shingleSize, FieldType fieldType) {
super(name, true, false, new TextSearchInfo(fieldType), Collections.emptyMap());
super(name, true, false, new TextSearchInfo(fieldType, null), Collections.emptyMap());
this.shingleSize = shingleSize;
}
@ -689,7 +700,8 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper {
this.shingleFields[i] = (ShingleFieldMapper) this.shingleFields[i].merge(m.shingleFields[i]);
}
}
if (Objects.equals(this.fieldType().similarity(), other.fieldType().similarity()) == false) {
if (Objects.equals(this.fieldType().getTextSearchInfo().getSimilarity(),
other.fieldType().getTextSearchInfo().getSimilarity()) == false) {
conflicts.add("mapper [" + name() + "] has different [similarity] settings");
}
}
@ -719,6 +731,11 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper {
protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
super.doXContentBody(builder, includeDefaults, params);
doXContentAnalyzers(builder, includeDefaults);
if (fieldType().getTextSearchInfo().getSimilarity() != null) {
builder.field("similarity", fieldType().getTextSearchInfo().getSimilarity().name());
} else if (includeDefaults) {
builder.field("similarity", SimilarityService.DEFAULT_SIMILARITY);
}
builder.field("max_shingle_size", maxShingleSize);
}

7
modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java
View File

@ -32,6 +32,8 @@ import org.apache.lucene.search.MultiPhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SynonymQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.BooleanSimilarity;
import org.apache.lucene.search.spans.FieldMaskingSpanQuery;
import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
@ -56,6 +58,7 @@ import org.elasticsearch.index.query.MatchPhrasePrefixQueryBuilder;
import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.similarity.SimilarityProvider;
import org.elasticsearch.plugins.Plugin;
import org.hamcrest.Matcher;
import org.hamcrest.Matchers;
@ -88,6 +91,10 @@ public class SearchAsYouTypeFieldMapperTests extends FieldMapperTestCase<SearchA
a.maxShingleSize(3);
b.maxShingleSize(2);
});
addModifier("similarity", false, (a, b) -> {
a.similarity(new SimilarityProvider("BM25", new BM25Similarity()));
b.similarity(new SimilarityProvider("boolean", new BooleanSimilarity()));
});
}
@Override

6
modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java
View File

@ -51,7 +51,7 @@ public class SearchAsYouTypeFieldTypeTests extends FieldTypeTestCase<MappedField
@Override
protected SearchAsYouTypeFieldType createDefaultFieldType(String name, Map<String, String> meta) {
final SearchAsYouTypeFieldType fieldType = new SearchAsYouTypeFieldType(name, Defaults.FIELD_TYPE, meta);
final SearchAsYouTypeFieldType fieldType = new SearchAsYouTypeFieldType(name, Defaults.FIELD_TYPE, null, meta);
fieldType.setPrefixField(new PrefixFieldType(NAME, Defaults.FIELD_TYPE, Defaults.MIN_GRAM, Defaults.MAX_GRAM));
fieldType.setShingleFields(new ShingleFieldType[] { new ShingleFieldType(fieldType.name(), 2, Defaults.FIELD_TYPE) });
return fieldType;
@ -62,7 +62,7 @@ public class SearchAsYouTypeFieldTypeTests extends FieldTypeTestCase<MappedField
assertThat(fieldType.termQuery("foo", null), equalTo(new TermQuery(new Term(NAME, "foo"))));
SearchAsYouTypeFieldType unsearchable = new SearchAsYouTypeFieldType(NAME, UNSEARCHABLE, Collections.emptyMap());
SearchAsYouTypeFieldType unsearchable = new SearchAsYouTypeFieldType(NAME, UNSEARCHABLE, null, Collections.emptyMap());
final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("foo", null));
assertThat(e.getMessage(), equalTo("Cannot search on field [" + NAME + "] since it is not indexed."));
}
@ -73,7 +73,7 @@ public class SearchAsYouTypeFieldTypeTests extends FieldTypeTestCase<MappedField
assertThat(fieldType.termsQuery(asList("foo", "bar"), null),
equalTo(new TermInSetQuery(NAME, asList(new BytesRef("foo"), new BytesRef("bar")))));
SearchAsYouTypeFieldType unsearchable = new SearchAsYouTypeFieldType(NAME, UNSEARCHABLE, Collections.emptyMap());
SearchAsYouTypeFieldType unsearchable = new SearchAsYouTypeFieldType(NAME, UNSEARCHABLE, null, Collections.emptyMap());
final IllegalArgumentException e =
expectThrows(IllegalArgumentException.class, () -> unsearchable.termsQuery(asList("foo", "bar"), null));
assertThat(e.getMessage(), equalTo("Cannot search on field [" + NAME + "] since it is not indexed."));

3
server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
View File

@ -195,7 +195,7 @@ public class CompletionFieldMapper extends FieldMapper {
private ContextMappings contextMappings = null;
public CompletionFieldType(String name, FieldType luceneFieldType, Map<String, String> meta) {
super(name, true, false, new TextSearchInfo(luceneFieldType), meta);
super(name, true, false, new TextSearchInfo(luceneFieldType, null), meta);
}
public CompletionFieldType(String name) {
@ -402,7 +402,6 @@ public class CompletionFieldMapper extends FieldMapper {
ft.setIndexAnalyzer(indexAnalyzer);
ft.setSearchAnalyzer(searchAnalyzer);
ft.setSearchQuoteAnalyzer(searchQuoteAnalyzer);
ft.setSimilarity(similarity);
return new CompletionFieldMapper(name, this.fieldType, ft,
multiFieldsBuilder.build(this, context), copyTo, maxInputLength);
}

18
server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
View File

@ -32,8 +32,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.support.AbstractXContentParser;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.FieldNamesFieldMapper.FieldNamesFieldType;
import org.elasticsearch.index.similarity.SimilarityProvider;
import org.elasticsearch.index.similarity.SimilarityService;
import java.io.IOException;
import java.util.ArrayList;
@ -70,7 +68,6 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
protected NamedAnalyzer indexAnalyzer;
protected NamedAnalyzer searchAnalyzer;
protected NamedAnalyzer searchQuoteAnalyzer;
protected SimilarityProvider similarity;
protected Builder(String name, FieldType fieldType) {
super(name);
@ -159,11 +156,6 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
return builder;
}
public T similarity(SimilarityProvider similarity) {
this.similarity = similarity;
return builder;
}
public T setEagerGlobalOrdinals(boolean eagerGlobalOrdinals) {
this.eagerGlobalOrdinals = eagerGlobalOrdinals;
return builder;
@ -374,10 +366,6 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
} else if (mappedFieldType.indexAnalyzer().name().equals(otherm.indexAnalyzer().name()) == false) {
conflicts.add("mapper [" + name() + "] has different [analyzer]");
}
if (Objects.equals(mappedFieldType.similarity(), otherm.similarity()) == false) {
conflicts.add("mapper [" + name() + "] has different [similarity]");
}
}
/**
@ -423,12 +411,6 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
builder.field("store", fieldType.stored());
}
if (fieldType().similarity() != null) {
builder.field("similarity", fieldType().similarity().name());
} else if (includeDefaults) {
builder.field("similarity", SimilarityService.DEFAULT_SIMILARITY);
}
multiFields.toXContent(builder, params);
copyTo.toXContent(builder, params);

27
server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
View File

@ -43,6 +43,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.similarity.SimilarityProvider;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
import java.io.IOException;
@ -52,6 +53,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import static org.elasticsearch.index.mapper.TypeParsers.checkNull;
import static org.elasticsearch.index.mapper.TypeParsers.parseField;
/**
@ -97,6 +99,7 @@ public final class KeywordFieldMapper extends FieldMapper {
private String normalizerName = "default";
private boolean eagerGlobalOrdinals = Defaults.EAGER_GLOBAL_ORDINALS;
private boolean splitQueriesOnWhitespace = Defaults.SPLIT_QUERIES_ON_WHITESPACE;
private SimilarityProvider similarity;
public Builder(String name) {
super(name, Defaults.FIELD_TYPE);
@ -109,6 +112,10 @@ public final class KeywordFieldMapper extends FieldMapper {
return builder;
}
public void similarity(SimilarityProvider similarity) {
this.similarity = similarity;
}
public Builder ignoreAbove(int ignoreAbove) {
if (ignoreAbove < 0) {
throw new IllegalArgumentException("[ignore_above] must be positive, got " + ignoreAbove);
@ -181,11 +188,11 @@ public final class KeywordFieldMapper extends FieldMapper {
@Override
public Mapper.Builder<?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder(name);
parseField(builder, name, node, parserContext);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
Map.Entry<String, Object> entry = iterator.next();
String propName = entry.getKey();
Object propNode = entry.getValue();
checkNull(propName, propNode);
if (propName.equals("null_value")) {
if (propNode == null) {
throw new MapperParsingException("Property [null_value] cannot be null.");
@ -209,8 +216,13 @@ public final class KeywordFieldMapper extends FieldMapper {
} else if (propName.equals("split_queries_on_whitespace")) {
builder.splitQueriesOnWhitespace(XContentMapValues.nodeBooleanValue(propNode, "split_queries_on_whitespace"));
iterator.remove();
} else if (propName.equals("similarity")) {
SimilarityProvider similarityProvider = TypeParsers.resolveSimilarity(parserContext, name, propNode.toString());
builder.similarity(similarityProvider);
iterator.remove();
}
}
parseField(builder, name, node, parserContext);
return builder;
}
}
@ -223,12 +235,11 @@ public final class KeywordFieldMapper extends FieldMapper {
boolean eagerGlobalOrdinals, NamedAnalyzer normalizer, NamedAnalyzer searchAnalyzer,
SimilarityProvider similarity, Map<String, String> meta, float boost) {
super(name, fieldType.indexOptions() != IndexOptions.NONE,
hasDocValues, new TextSearchInfo(fieldType), meta);
hasDocValues, new TextSearchInfo(fieldType, similarity), meta);
this.hasNorms = fieldType.omitNorms() == false;
setEagerGlobalOrdinals(eagerGlobalOrdinals);
setIndexAnalyzer(normalizer);
setSearchAnalyzer(searchAnalyzer);
setSimilarity(similarity);
setBoost(boost);
}
@ -422,6 +433,10 @@ public final class KeywordFieldMapper extends FieldMapper {
if (Objects.equals(fieldType().indexAnalyzer(), k.fieldType().indexAnalyzer()) == false) {
conflicts.add("mapper [" + name() + "] has different [normalizer]");
}
if (Objects.equals(mappedFieldType.getTextSearchInfo().getSimilarity(),
k.fieldType().getTextSearchInfo().getSimilarity()) == false) {
conflicts.add("mapper [" + name() + "] has different [similarity]");
}
this.ignoreAbove = k.ignoreAbove;
this.splitQueriesOnWhitespace = k.splitQueriesOnWhitespace;
this.fieldType().setSearchAnalyzer(k.fieldType().searchAnalyzer());
@ -456,6 +471,12 @@ public final class KeywordFieldMapper extends FieldMapper {
builder.field("normalizer", "default");
}
if (fieldType().getTextSearchInfo().getSimilarity() != null) {
builder.field("similarity", fieldType().getTextSearchInfo().getSimilarity().name());
} else if (includeDefaults) {
builder.field("similarity", SimilarityService.DEFAULT_SIMILARITY);
}
if (includeDefaults || splitQueriesOnWhitespace) {
builder.field("split_queries_on_whitespace", splitQueriesOnWhitespace);
}

14
server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
View File

@ -46,7 +46,6 @@ import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.query.QueryRewriteContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.query.QueryShardException;
import org.elasticsearch.index.similarity.SimilarityProvider;
import org.elasticsearch.search.DocValueFormat;
import java.io.IOException;
@ -68,7 +67,6 @@ public abstract class MappedFieldType {
private NamedAnalyzer indexAnalyzer;
private NamedAnalyzer searchAnalyzer;
private NamedAnalyzer searchQuoteAnalyzer;
private SimilarityProvider similarity;
private boolean eagerGlobalOrdinals;
private Map<String, String> meta;
@ -80,7 +78,6 @@ public abstract class MappedFieldType {
this.indexAnalyzer = ref.indexAnalyzer();
this.searchAnalyzer = ref.searchAnalyzer();
this.searchQuoteAnalyzer = ref.searchQuoteAnalyzer;
this.similarity = ref.similarity();
this.eagerGlobalOrdinals = ref.eagerGlobalOrdinals;
this.meta = ref.meta;
this.textSearchInfo = ref.textSearchInfo;
@ -125,14 +122,13 @@ public abstract class MappedFieldType {
Objects.equals(searchAnalyzer, fieldType.searchAnalyzer) &&
Objects.equals(searchQuoteAnalyzer(), fieldType.searchQuoteAnalyzer()) &&
Objects.equals(eagerGlobalOrdinals, fieldType.eagerGlobalOrdinals) &&
Objects.equals(similarity, fieldType.similarity) &&
Objects.equals(meta, fieldType.meta);
}
@Override
public int hashCode() {
return Objects.hash(name, boost, docValues, indexAnalyzer, searchAnalyzer, searchQuoteAnalyzer,
eagerGlobalOrdinals, similarity == null ? null : similarity.name(), meta);
eagerGlobalOrdinals, meta);
}
// TODO: we need to override freeze() and add safety checks that all settings are actually set
@ -180,14 +176,6 @@ public abstract class MappedFieldType {
this.searchQuoteAnalyzer = analyzer;
}
public SimilarityProvider similarity() {
return similarity;
}
public void setSimilarity(SimilarityProvider similarity) {
this.similarity = similarity;
}
/** Given a value that comes from the stored fields API, convert it to the
* expected type. For instance a date field would store dates as longs and
* format it back to a string in this method. */

35
server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
View File

@ -68,6 +68,8 @@ import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
import org.elasticsearch.index.query.IntervalBuilder;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.similarity.SimilarityProvider;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
import java.io.IOException;
@ -80,6 +82,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.function.IntPredicate;
import static org.elasticsearch.index.mapper.TypeParsers.checkNull;
import static org.elasticsearch.index.mapper.TypeParsers.parseTextField;
/** A {@link FieldMapper} for full-text fields. */
@ -126,6 +129,7 @@ public class TextFieldMapper extends FieldMapper {
private double fielddataMinFreq = Defaults.FIELDDATA_MIN_FREQUENCY;
private double fielddataMaxFreq = Defaults.FIELDDATA_MAX_FREQUENCY;
private int fielddataMinSegSize = Defaults.FIELDDATA_MIN_SEGMENT_SIZE;
private SimilarityProvider similarity;
public Builder(String name) {
super(name, Defaults.FIELD_TYPE);
@ -150,6 +154,10 @@ public class TextFieldMapper extends FieldMapper {
return builder;
}
public void similarity(SimilarityProvider similarity) {
this.similarity = similarity;
}
@Override
public Builder docValues(boolean docValues) {
if (docValues) {
@ -187,12 +195,11 @@ public class TextFieldMapper extends FieldMapper {
}
private TextFieldType buildFieldType(BuilderContext context) {
TextFieldType ft = new TextFieldType(buildFullName(context), fieldType, meta);
TextFieldType ft = new TextFieldType(buildFullName(context), fieldType, similarity, meta);
ft.setIndexAnalyzer(indexAnalyzer);
ft.setSearchAnalyzer(searchAnalyzer);
ft.setSearchQuoteAnalyzer(searchQuoteAnalyzer);
ft.setEagerGlobalOrdinals(eagerGlobalOrdinals);
ft.setSimilarity(similarity);
if (fielddata) {
ft.setFielddata(true);
ft.setFielddataMinFrequency(fielddataMinFreq);
@ -275,11 +282,11 @@ public class TextFieldMapper extends FieldMapper {
builder.indexAnalyzer(parserContext.getIndexAnalyzers().getDefaultIndexAnalyzer());
builder.searchAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchAnalyzer());
builder.searchQuoteAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer());
parseTextField(builder, fieldName, node, parserContext);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
Map.Entry<String, Object> entry = iterator.next();
String propName = entry.getKey();
Object propNode = entry.getValue();
checkNull(propName, propNode);
if (propName.equals("position_increment_gap")) {
int newPositionIncrementGap = XContentMapValues.nodeIntegerValue(propNode, -1);
builder.positionIncrementGap(newPositionIncrementGap);
@ -310,8 +317,13 @@ public class TextFieldMapper extends FieldMapper {
} else if (propName.equals("index_phrases")) {
builder.indexPhrases(XContentMapValues.nodeBooleanValue(propNode, "index_phrases"));
iterator.remove();
} else if (propName.equals("similarity")) {
SimilarityProvider similarityProvider = TypeParsers.resolveSimilarity(parserContext, fieldName, propNode.toString());
builder.similarity(similarityProvider);
iterator.remove();
}
}
parseTextField(builder, fieldName, node, parserContext);
return builder;
}
}
@ -558,9 +570,9 @@ public class TextFieldMapper extends FieldMapper {
private boolean indexPhrases = false;
private final FieldType indexedFieldType;
public TextFieldType(String name, FieldType indexedFieldType, Map<String, String> meta) {
public TextFieldType(String name, FieldType indexedFieldType, SimilarityProvider similarity, Map<String, String> meta) {
super(name, indexedFieldType.indexOptions() != IndexOptions.NONE, false,
new TextSearchInfo(indexedFieldType), meta);
new TextSearchInfo(indexedFieldType, similarity), meta);
this.indexedFieldType = indexedFieldType;
fielddata = false;
fielddataMinFrequency = Defaults.FIELDDATA_MIN_FREQUENCY;
@ -569,13 +581,13 @@ public class TextFieldMapper extends FieldMapper {
}
public TextFieldType(String name, boolean indexed, Map<String, String> meta) {
super(name, indexed, false, new TextSearchInfo(Defaults.FIELD_TYPE), meta);
super(name, indexed, false, new TextSearchInfo(Defaults.FIELD_TYPE, null), meta);
this.indexedFieldType = Defaults.FIELD_TYPE;
fielddata = false;
}
public TextFieldType(String name) {
this(name, Defaults.FIELD_TYPE, Collections.emptyMap());
this(name, Defaults.FIELD_TYPE, null, Collections.emptyMap());
}
protected TextFieldType(TextFieldType ref) {
@ -892,7 +904,8 @@ public class TextFieldMapper extends FieldMapper {
@Override
protected void mergeOptions(FieldMapper other, List<String> conflicts) {
TextFieldMapper mw = (TextFieldMapper) other;
if (Objects.equals(mw.fieldType().similarity(), this.fieldType().similarity()) == false) {
if (Objects.equals(mw.fieldType().getTextSearchInfo().getSimilarity(),
this.fieldType().getTextSearchInfo().getSimilarity()) == false) {
conflicts.add("mapper [" + name() + "] has different [similarity] settings");
}
if (mw.fieldType().indexPhrases != this.fieldType().indexPhrases) {
@ -935,7 +948,11 @@ public class TextFieldMapper extends FieldMapper {
builder.field("norms", fieldType.omitNorms() == false);
}
doXContentAnalyzers(builder, includeDefaults);
if (fieldType().getTextSearchInfo().getSimilarity() != null) {
builder.field("similarity", fieldType().getTextSearchInfo().getSimilarity().name());
} else if (includeDefaults) {
builder.field("similarity", SimilarityService.DEFAULT_SIMILARITY);
}
if (includeDefaults || fieldType().eagerGlobalOrdinals()) {
builder.field("eager_global_ordinals", fieldType().eagerGlobalOrdinals());
}

19
server/src/main/java/org/elasticsearch/index/mapper/TextSearchInfo.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexOptions;
import org.elasticsearch.index.similarity.SimilarityProvider;
/**
* Encapsulates information about how to perform text searches over a field
@ -39,20 +40,30 @@ public class TextSearchInfo {
*
* Note that the results of {@link #isStored()} for this may not be accurate
*/
public static final TextSearchInfo SIMPLE_MATCH_ONLY = new TextSearchInfo(SIMPLE_MATCH_ONLY_FIELD_TYPE);
public static final TextSearchInfo SIMPLE_MATCH_ONLY = new TextSearchInfo(SIMPLE_MATCH_ONLY_FIELD_TYPE, null);
/**
* Specifies that this field does not support text searching of any kind
*/
public static final TextSearchInfo NONE = new TextSearchInfo(SIMPLE_MATCH_ONLY_FIELD_TYPE);
public static final TextSearchInfo NONE = new TextSearchInfo(SIMPLE_MATCH_ONLY_FIELD_TYPE, null);
private final FieldType luceneFieldType;
private final SimilarityProvider similarity;
/**
* Create a TextSearchInfo by wrapping a lucene FieldType
* Create a new TextSearchInfo
*
* @param luceneFieldType the lucene {@link FieldType} of the field to be searched
* @param similarity defines which Similarity to use when searching. If set to {@code null}
* then the default Similarity will be used.
*/
public TextSearchInfo(FieldType luceneFieldType) {
public TextSearchInfo(FieldType luceneFieldType, SimilarityProvider similarity) {
this.luceneFieldType = luceneFieldType;
this.similarity = similarity;
}
public SimilarityProvider getSimilarity() {
return similarity;
}
/**

24
server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
View File

@ -168,6 +168,16 @@ public class TypeParsers {
}
}
public static void checkNull(String propName, Object propNode) {
if (false == propName.equals("null_value") && propNode == null) {
/*
* No properties *except* null_value are allowed to have null. So we catch it here and tell the user something useful rather
* than send them a null pointer exception later.
*/
throw new MapperParsingException("[" + propName + "] must not have a [null] value");
}
}
/**
* Parse the {@code meta} key of the mapping.
*/
@ -224,13 +234,7 @@ public class TypeParsers {
Map.Entry<String, Object> entry = iterator.next();
final String propName = entry.getKey();
final Object propNode = entry.getValue();
if (false == propName.equals("null_value") && propNode == null) {
/*
* No properties *except* null_value are allowed to have null. So we catch it here and tell the user something useful rather
* than send them a null pointer exception later.
*/
throw new MapperParsingException("[" + propName + "] must not have a [null] value");
}
checkNull(propName, propNode);
if (propName.equals("store")) {
builder.store(XContentMapValues.nodeBooleanValue(propNode, name + ".store"));
iterator.remove();
@ -247,8 +251,8 @@ public class TypeParsers {
builder.indexOptions(nodeIndexOptionValue(propNode));
iterator.remove();
} else if (propName.equals("similarity")) {
SimilarityProvider similarityProvider = resolveSimilarity(parserContext, name, propNode.toString());
builder.similarity(similarityProvider);
deprecationLogger.deprecatedAndMaybeLog("similarity",
"The [similarity] parameter has no effect on field [" + name + "] and will be removed in 8.0");
iterator.remove();
} else if (parseMultiField(builder, name, parserContext, propName, propNode)) {
iterator.remove();
@ -384,7 +388,7 @@ public class TypeParsers {
builder.copyTo(copyToBuilder.build());
}
private static SimilarityProvider resolveSimilarity(Mapper.TypeParser.ParserContext parserContext, String name, String value) {
public static SimilarityProvider resolveSimilarity(Mapper.TypeParser.ParserContext parserContext, String name, String value) {
SimilarityProvider similarityProvider = parserContext.getSimilarity(value);
if (similarityProvider == null) {
throw new MapperParsingException("Unknown Similarity type [" + value + "] for field [" + name + "]");

3
server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
View File

@ -195,7 +195,8 @@ public final class SimilarityService extends AbstractIndexComponent {
@Override
public Similarity get(String name) {
MappedFieldType fieldType = mapperService.fieldType(name);
return (fieldType != null && fieldType.similarity() != null) ? fieldType.similarity().get() : defaultSimilarity;
return (fieldType != null && fieldType.getTextSearchInfo().getSimilarity() != null)
? fieldType.getTextSearchInfo().getSimilarity().get() : defaultSimilarity;
}
}

7
server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
View File

@ -25,6 +25,8 @@ import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.IndexableFieldType;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.BooleanSimilarity;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
@ -36,6 +38,7 @@ import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.analysis.PreConfiguredTokenFilter;
import org.elasticsearch.index.analysis.TokenizerFactory;
import org.elasticsearch.index.mapper.MapperService.MergeReason;
import org.elasticsearch.index.similarity.SimilarityProvider;
import org.elasticsearch.index.termvectors.TermVectorsService;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.plugins.AnalysisPlugin;
@ -118,6 +121,10 @@ public class KeywordFieldMapperTests extends FieldMapperTestCase<KeywordFieldMap
a.indexOptions(IndexOptions.DOCS);
b.indexOptions(IndexOptions.DOCS_AND_FREQS);
});
addModifier("similarity", false, (a, b) -> {
a.similarity(new SimilarityProvider("BM25", new BM25Similarity()));
b.similarity(new SimilarityProvider("boolean", new BooleanSimilarity()));
});
}
public void testDefaults() throws Exception {

11
server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
View File

@ -479,6 +479,17 @@ public class NumberFieldMapperTests extends AbstractNumericFieldMapperTestCase<N
assertTrue(response.status() == RestStatus.CREATED);
}
public void testDeprecatedSimilarityParameter() throws Exception {
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("field")
.field("type", "long")
.field("similarity", "bm25")
.endObject().endObject().endObject().endObject());
parser.parse("type", new CompressedXContent(mapping));
assertWarnings("The [similarity] parameter has no effect on field [field] and will be removed in 8.0");
}
private void parseRequest(NumberType type, BytesReference content) throws IOException {
createDocumentMapper(type).parse(new SourceToParse("test", "type", "1", content, XContentType.JSON));
}

7
server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
View File

@ -42,6 +42,8 @@ import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SynonymQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.BooleanSimilarity;
import org.apache.lucene.search.spans.FieldMaskingSpanQuery;
import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanOrQuery;
@ -72,6 +74,7 @@ import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.search.MatchQuery;
import org.elasticsearch.index.seqno.SequenceNumbers;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.similarity.SimilarityProvider;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.InternalSettingsPlugin;
import org.junit.Before;
@ -124,6 +127,10 @@ public class TextFieldMapperTests extends FieldMapperTestCase<TextFieldMapper.Bu
a.indexOptions(IndexOptions.DOCS_AND_FREQS);
b.indexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
});
addModifier("similarity", false, (a, b) -> {
a.similarity(new SimilarityProvider("BM25", new BM25Similarity()));
b.similarity(new SimilarityProvider("boolean", new BooleanSimilarity()));
});
}
@Override

6
server/src/test/java/org/elasticsearch/index/similarity/LegacySimilarityTests.java
View File

@ -83,9 +83,11 @@ public class LegacySimilarityTests extends ESSingleNodeTestCase {
.put("index.similarity.my_similarity.discount_overlaps", false)
.build();
final MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService();
assertThat(mapperService.fieldType("field1").similarity().get(), instanceOf(ClassicSimilarity.class));
assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(),
instanceOf(ClassicSimilarity.class));
final ClassicSimilarity similarity = (ClassicSimilarity) mapperService.fieldType("field1").similarity().get();
final ClassicSimilarity similarity
= (ClassicSimilarity) mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get();
assertThat(similarity.getDiscountOverlaps(), equalTo(false));
}
}

30
server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java
View File

@ -94,9 +94,10 @@ public class SimilarityTests extends ESSingleNodeTestCase {
.put("index.similarity.my_similarity.discount_overlaps", false)
.build();
MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService();
assertThat(mapperService.fieldType("field1").similarity().get(), instanceOf(LegacyBM25Similarity.class));
assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(), instanceOf(LegacyBM25Similarity.class));
LegacyBM25Similarity similarity = (LegacyBM25Similarity) mapperService.fieldType("field1").similarity().get();
LegacyBM25Similarity similarity
= (LegacyBM25Similarity) mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get();
assertThat(similarity.getK1(), equalTo(2.0f));
assertThat(similarity.getB(), equalTo(0.5f));
assertThat(similarity.getDiscountOverlaps(), equalTo(false));
@ -110,7 +111,7 @@ public class SimilarityTests extends ESSingleNodeTestCase {
.endObject().endObject();
MapperService mapperService = createIndex("foo", Settings.EMPTY, "type", mapping).mapperService();
assertThat(mapperService.fieldType("field1").similarity().get(), instanceOf(BooleanSimilarity.class));
assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(), instanceOf(BooleanSimilarity.class));
}
public void testResolveSimilaritiesFromMapping_DFR() throws IOException {
@ -128,9 +129,9 @@ public class SimilarityTests extends ESSingleNodeTestCase {
.put("index.similarity.my_similarity.normalization.h2.c", 3f)
.build();
MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService();
assertThat(mapperService.fieldType("field1").similarity().get(), instanceOf(DFRSimilarity.class));
assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(), instanceOf(DFRSimilarity.class));
DFRSimilarity similarity = (DFRSimilarity) mapperService.fieldType("field1").similarity().get();
DFRSimilarity similarity = (DFRSimilarity) mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get();
assertThat(similarity.getBasicModel(), instanceOf(BasicModelG.class));
assertThat(similarity.getAfterEffect(), instanceOf(AfterEffectL.class));
assertThat(similarity.getNormalization(), instanceOf(NormalizationH2.class));
@ -152,9 +153,9 @@ public class SimilarityTests extends ESSingleNodeTestCase {
.put("index.similarity.my_similarity.normalization.h2.c", 3f)
.build();
MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService();
assertThat(mapperService.fieldType("field1").similarity().get(), instanceOf(IBSimilarity.class));
assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(), instanceOf(IBSimilarity.class));
IBSimilarity similarity = (IBSimilarity) mapperService.fieldType("field1").similarity().get();
IBSimilarity similarity = (IBSimilarity) mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get();
assertThat(similarity.getDistribution(), instanceOf(DistributionSPL.class));
assertThat(similarity.getLambda(), instanceOf(LambdaTTF.class));
assertThat(similarity.getNormalization(), instanceOf(NormalizationH2.class));
@ -175,8 +176,8 @@ public class SimilarityTests extends ESSingleNodeTestCase {
MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService();
MappedFieldType fieldType = mapperService.fieldType("field1");
assertThat(fieldType.similarity().get(), instanceOf(DFISimilarity.class));
DFISimilarity similarity = (DFISimilarity) fieldType.similarity().get();
assertThat(fieldType.getTextSearchInfo().getSimilarity().get(), instanceOf(DFISimilarity.class));
DFISimilarity similarity = (DFISimilarity) fieldType.getTextSearchInfo().getSimilarity().get();
assertThat(similarity.getIndependence(), instanceOf(IndependenceChiSquared.class));
}
@ -193,9 +194,10 @@ public class SimilarityTests extends ESSingleNodeTestCase {
.build();
MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService();
assertThat(mapperService.fieldType("field1").similarity().get(), instanceOf(LMDirichletSimilarity.class));
assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(), instanceOf(LMDirichletSimilarity.class));
LMDirichletSimilarity similarity = (LMDirichletSimilarity) mapperService.fieldType("field1").similarity().get();
LMDirichletSimilarity similarity
= (LMDirichletSimilarity) mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get();
assertThat(similarity.getMu(), equalTo(3000f));
}
@ -211,9 +213,11 @@ public class SimilarityTests extends ESSingleNodeTestCase {
.put("index.similarity.my_similarity.lambda", 0.7f)
.build();
MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService();
assertThat(mapperService.fieldType("field1").similarity().get(), instanceOf(LMJelinekMercerSimilarity.class));
assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(),
instanceOf(LMJelinekMercerSimilarity.class));
LMJelinekMercerSimilarity similarity = (LMJelinekMercerSimilarity) mapperService.fieldType("field1").similarity().get();
LMJelinekMercerSimilarity similarity
= (LMJelinekMercerSimilarity) mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get();
assertThat(similarity.getLambda(), equalTo(0.7f));
}

7
test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java
View File

@ -22,8 +22,6 @@ package org.elasticsearch.index.mapper;
import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.BooleanSimilarity;
import org.elasticsearch.Version;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.compress.CompressedXContent;
@ -34,7 +32,6 @@ import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.similarity.SimilarityProvider;
import org.elasticsearch.test.ESSingleNodeTestCase;
import java.io.IOException;
@ -105,10 +102,6 @@ public abstract class FieldMapperTestCase<T extends FieldMapper.Builder<?>> exte
a.searchQuoteAnalyzer(new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer()));
a.searchQuoteAnalyzer(new NamedAnalyzer("whitespace", AnalyzerScope.INDEX, new WhitespaceAnalyzer()));
}),
new Modifier("similarity", false, (a, b) -> {
a.similarity(new SimilarityProvider("BM25", new BM25Similarity()));
b.similarity(new SimilarityProvider("boolean", new BooleanSimilarity()));
}),
new Modifier("store", false, (a, b) -> {
a.store(true);
b.store(false);

8
x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
View File

@ -63,7 +63,6 @@ import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.ParseContext.Document;
import org.elasticsearch.index.mapper.TextSearchInfo;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.similarity.SimilarityProvider;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
@ -145,11 +144,6 @@ public class WildcardFieldMapper extends FieldMapper {
return this;
}
@Override
public Builder similarity(SimilarityProvider similarity) {
throw new MapperParsingException("The field [" + name + "] cannot have custom similarities");
}
@Override
public Builder index(boolean index) {
if (index == false) {
@ -217,7 +211,7 @@ public class WildcardFieldMapper extends FieldMapper {
static Analyzer lowercaseNormalizer = new LowercaseNormalizer();
public WildcardFieldType(String name, FieldType fieldType, Map<String, String> meta) {
super(name, true, true, new TextSearchInfo(fieldType), meta);
super(name, true, true, new TextSearchInfo(fieldType, null), meta);
setIndexAnalyzer(WILDCARD_ANALYZER);
setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
}