honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add per-field Similarity support

This commit is contained in:
Chris Male 2012-11-21 12:44:59 +13:00
parent 4e8a9008b7
commit 9e2469e04f
53 changed files with 1138 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java
View File

@ -1331,7 +1331,7 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine {
config.setIndexDeletionPolicy(deletionPolicy);
config.setMergeScheduler(mergeScheduler.newMergeScheduler());
config.setMergePolicy(mergePolicyProvider.newMergePolicy());
config.setSimilarity(similarityService.defaultIndexSimilarity());
config.setSimilarity(similarityService.similarity());
config.setRAMBufferSizeMB(indexingBufferSize.mbFrac());
config.setTermIndexInterval(termIndexInterval);
config.setReaderTermsIndexDivisor(termIndexDivisor);
@ -1478,7 +1478,7 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine {
@Override
public IndexSearcher newSearcher(IndexReader reader) throws IOException {
IndexSearcher searcher = new IndexSearcher(reader);
searcher.setSimilarity(similarityService.defaultSearchSimilarity());
searcher.setSimilarity(similarityService.similarity());
if (warmer != null) {
// we need to pass a custom searcher that does not release anything on Engine.Search Release,
// we will release explicitly

15
src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
View File

@ -45,6 +45,7 @@ import org.elasticsearch.index.mapper.multifield.MultiFieldMapper;
import org.elasticsearch.index.mapper.object.ObjectMapper;
import org.elasticsearch.index.mapper.object.RootObjectMapper;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.similarity.SimilarityLookupService;
import java.io.IOException;
import java.util.Map;
@ -58,6 +59,7 @@ public class DocumentMapperParser extends AbstractIndexComponent {
final AnalysisService analysisService;
private final PostingsFormatService postingsFormatService;
private final SimilarityLookupService similarityLookupService;
private final RootObjectMapper.TypeParser rootObjectTypeParser = new RootObjectMapper.TypeParser();
@ -66,14 +68,17 @@ public class DocumentMapperParser extends AbstractIndexComponent {
private volatile ImmutableMap<String, Mapper.TypeParser> typeParsers;
private volatile ImmutableMap<String, Mapper.TypeParser> rootTypeParsers;
public DocumentMapperParser(Index index, AnalysisService analysisService, PostingsFormatService postingsFormatService) {
this(index, ImmutableSettings.Builder.EMPTY_SETTINGS, analysisService, postingsFormatService);
public DocumentMapperParser(Index index, AnalysisService analysisService, PostingsFormatService postingsFormatService,
SimilarityLookupService similarityLookupService) {
this(index, ImmutableSettings.Builder.EMPTY_SETTINGS, analysisService, postingsFormatService, similarityLookupService);
}
public DocumentMapperParser(Index index, @IndexSettings Settings indexSettings, AnalysisService analysisService, PostingsFormatService postingsFormatService) {
public DocumentMapperParser(Index index, @IndexSettings Settings indexSettings, AnalysisService analysisService,
PostingsFormatService postingsFormatService, SimilarityLookupService similarityLookupService) {
super(index, indexSettings);
this.analysisService = analysisService;
this.postingsFormatService = postingsFormatService;
this.similarityLookupService = similarityLookupService;
MapBuilder<String, Mapper.TypeParser> typeParsersBuilder = new MapBuilder<String, Mapper.TypeParser>()
.put(ByteFieldMapper.CONTENT_TYPE, new ByteFieldMapper.TypeParser())
.put(ShortFieldMapper.CONTENT_TYPE, new ShortFieldMapper.TypeParser())
@ -132,7 +137,7 @@ public class DocumentMapperParser extends AbstractIndexComponent {
}
public Mapper.TypeParser.ParserContext parserContext() {
return new Mapper.TypeParser.ParserContext(postingsFormatService, analysisService, typeParsers);
return new Mapper.TypeParser.ParserContext(postingsFormatService, analysisService, similarityLookupService, typeParsers);
}
public DocumentMapper parse(String source) throws MapperParsingException {
@ -166,7 +171,7 @@ public class DocumentMapperParser extends AbstractIndexComponent {
}
}
Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext(postingsFormatService, analysisService, typeParsers);
Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext(postingsFormatService, analysisService, similarityLookupService, typeParsers);
DocumentMapper.Builder docBuilder = doc(index.name(), indexSettings, (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext));

6
src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
View File

@ -30,6 +30,7 @@ import org.elasticsearch.common.Nullable;
import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
import org.elasticsearch.index.field.data.FieldDataType;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.similarity.SimilarityProvider;
/**
*
@ -157,6 +158,11 @@ public interface FieldMapper<T> {
*/
Analyzer searchQuoteAnalyzer();
/**
* Similarity used for scoring queries on the field
*/
SimilarityProvider similarity();
/**
* Returns the value that will be used as a result for search. Can be only of specific types... .
*/

11
src/main/java/org/elasticsearch/index/mapper/Mapper.java
View File

@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.codec.postingsformat.PostingsFormatService;
import org.elasticsearch.index.similarity.SimilarityLookupService;
import java.io.IOException;
import java.util.Map;
@ -81,11 +82,15 @@ public interface Mapper extends ToXContent {
private final AnalysisService analysisService;
private final SimilarityLookupService similarityLookupService;
private final ImmutableMap<String, TypeParser> typeParsers;
public ParserContext(PostingsFormatService postingsFormatService, AnalysisService analysisService, ImmutableMap<String, TypeParser> typeParsers) {
public ParserContext(PostingsFormatService postingsFormatService, AnalysisService analysisService,
SimilarityLookupService similarityLookupService, ImmutableMap<String, TypeParser> typeParsers) {
this.postingsFormatService = postingsFormatService;
this.analysisService = analysisService;
this.similarityLookupService = similarityLookupService;
this.typeParsers = typeParsers;
}
@ -97,6 +102,10 @@ public interface Mapper extends ToXContent {
return postingsFormatService;
}
public SimilarityLookupService similarityLookupService() {
return similarityLookupService;
}
public TypeParser typeParser(String type) {
return typeParsers.get(Strings.toUnderscoreCase(type));
}

6
src/main/java/org/elasticsearch/index/mapper/MapperService.java
View File

@ -49,6 +49,7 @@ import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
import org.elasticsearch.index.mapper.object.ObjectMapper;
import org.elasticsearch.index.search.nested.NonNestedDocsFilter;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.similarity.SimilarityLookupService;
import org.elasticsearch.indices.InvalidTypeNameException;
import org.elasticsearch.indices.TypeMissingException;
@ -99,11 +100,12 @@ public class MapperService extends AbstractIndexComponent implements Iterable<Do
private final SmartIndexNameSearchQuoteAnalyzer searchQuoteAnalyzer;
@Inject
public MapperService(Index index, @IndexSettings Settings indexSettings, Environment environment, AnalysisService analysisService, PostingsFormatService postingsFormatService) {
public MapperService(Index index, @IndexSettings Settings indexSettings, Environment environment, AnalysisService analysisService,
PostingsFormatService postingsFormatService, SimilarityLookupService similarityLookupService) {
super(index, indexSettings);
this.analysisService = analysisService;
this.postingsFormatService = postingsFormatService;
this.documentParser = new DocumentMapperParser(index, indexSettings, analysisService, postingsFormatService);
this.documentParser = new DocumentMapperParser(index, indexSettings, analysisService, postingsFormatService, similarityLookupService);
this.searchAnalyzer = new SmartIndexNameSearchAnalyzer(analysisService.defaultSearchAnalyzer());
this.searchQuoteAnalyzer = new SmartIndexNameSearchQuoteAnalyzer(analysisService.defaultSearchQuoteAnalyzer());

32
src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java
View File

@ -37,6 +37,7 @@ import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
import org.elasticsearch.index.field.data.FieldDataType;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.similarity.SimilarityProvider;
import java.io.IOException;
@ -131,6 +132,11 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T>, Mapper {
public T searchAnalyzer(NamedAnalyzer searchAnalyzer) {
return super.searchAnalyzer(searchAnalyzer);
}
@Override
public T similarity(SimilarityProvider similarity) {
return super.similarity(similarity);
}
}
public abstract static class Builder<T extends Builder, Y extends AbstractFieldMapper> extends Mapper.Builder<T, Y> {
@ -144,6 +150,7 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T>, Mapper {
protected Boolean includeInAll;
protected boolean indexOptionsSet = false;
protected PostingsFormatProvider provider;
protected SimilarityProvider similarity;
protected Builder(String name, FieldType fieldType) {
super(name);
@ -230,6 +237,11 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T>, Mapper {
return builder;
}
protected T similarity(SimilarityProvider similarity) {
this.similarity = similarity;
return builder;
}
protected Names buildNames(BuilderContext context) {
return new Names(name, buildIndexName(context), indexName == null ? name : indexName, buildFullName(context), context.path().sourcePath());
}
@ -250,9 +262,10 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T>, Mapper {
protected final NamedAnalyzer indexAnalyzer;
protected final NamedAnalyzer searchAnalyzer;
protected PostingsFormatProvider postingsFormat;
protected final SimilarityProvider similarity;
protected AbstractFieldMapper(Names names, float boost, FieldType fieldType, NamedAnalyzer indexAnalyzer,
NamedAnalyzer searchAnalyzer, PostingsFormatProvider postingsFormat) {
NamedAnalyzer searchAnalyzer, PostingsFormatProvider postingsFormat, SimilarityProvider similarity) {
this.names = names;
this.boost = boost;
this.fieldType = fieldType;
@ -276,6 +289,7 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T>, Mapper {
}
}
this.postingsFormat = postingsFormat;
this.similarity = similarity;
}
@Nullable
@ -358,6 +372,11 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T>, Mapper {
return this.searchAnalyzer;
}
@Override
public SimilarityProvider similarity() {
return similarity;
}
@Override
public void parse(ParseContext context) throws IOException {
try {
@ -524,6 +543,17 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T>, Mapper {
} else if (!this.searchAnalyzer.name().equals(fieldMergeWith.searchAnalyzer.name())) {
mergeContext.addConflict("mapper [" + names.fullName() + "] has different search_analyzer");
}
if (this.similarity == null) {
if (fieldMergeWith.similarity() != null) {
mergeContext.addConflict("mapper [" + names.fullName() + "] has different similarity");
}
} else if (fieldMergeWith.similarity() == null) {
mergeContext.addConflict("mapper [" + names.fullName() + "] has different similarity");
} else if (!this.similarity().equals(fieldMergeWith.similarity())) {
mergeContext.addConflict("mapper [" + names.fullName() + "] has different similarity");
}
if (!mergeContext.mergeFlags().simulate()) {
// apply changeable values
this.boost = fieldMergeWith.boost;

2
src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
View File

@ -120,7 +120,7 @@ public class BinaryFieldMapper extends AbstractFieldMapper<byte[]> {
private long compressThreshold;
protected BinaryFieldMapper(Names names, FieldType fieldType, Boolean compress, long compressThreshold, PostingsFormatProvider provider) {
super(names, 1.0f, fieldType, null, null, provider);
super(names, 1.0f, fieldType, null, null, provider, null);
this.compress = compress;
this.compressThreshold = compressThreshold;
}

14
src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
View File

@ -32,6 +32,7 @@ import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.similarity.SimilarityProvider;
import java.io.IOException;
import java.util.Map;
@ -113,10 +114,14 @@ public class BooleanFieldMapper extends AbstractFieldMapper<Boolean> {
return super.indexName(indexName);
}
@Override
public Builder similarity(SimilarityProvider similarity) {
return super.similarity(similarity);
}
@Override
public BooleanFieldMapper build(BuilderContext context) {
return new BooleanFieldMapper(buildNames(context), boost, fieldType, nullValue, provider);
return new BooleanFieldMapper(buildNames(context), boost, fieldType, nullValue, provider, similarity);
}
}
@ -138,8 +143,8 @@ public class BooleanFieldMapper extends AbstractFieldMapper<Boolean> {
private Boolean nullValue;
protected BooleanFieldMapper(Names names, float boost, FieldType fieldType, Boolean nullValue, PostingsFormatProvider provider) {
super(names, boost, fieldType, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, provider);
protected BooleanFieldMapper(Names names, float boost, FieldType fieldType, Boolean nullValue, PostingsFormatProvider provider, SimilarityProvider similarity) {
super(names, boost, fieldType, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, provider, similarity);
this.nullValue = nullValue;
}
@ -238,6 +243,9 @@ public class BooleanFieldMapper extends AbstractFieldMapper<Boolean> {
if (indexOptions() != Defaults.BOOLEAN_FIELD_TYPE.indexOptions()) {
builder.field("index_options", indexOptionToString(indexOptions()));
}
if (similarity() != null) {
builder.field("similarity", similarity().name());
}
if (nullValue != null) {
builder.field("null_value", nullValue);
}

10
src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
View File

@ -42,6 +42,7 @@ import org.elasticsearch.index.field.data.FieldDataType;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
import org.elasticsearch.index.similarity.SimilarityProvider;
import java.io.IOException;
import java.util.Map;
@ -86,7 +87,7 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
ByteFieldMapper fieldMapper = new ByteFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, boost, fieldType, nullValue, ignoreMalformed(context),
provider);
provider, similarity);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@ -113,10 +114,10 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
private String nullValueAsString;
protected ByteFieldMapper(Names names, int precisionStep, String fuzzyFactor, float boost, FieldType fieldType,
Byte nullValue, Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider) {
Byte nullValue, Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity) {
super(names, precisionStep, fuzzyFactor, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_byte/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)),
new NamedAnalyzer("_byte/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider);
new NamedAnalyzer("_byte/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity);
this.nullValue = nullValue;
this.nullValueAsString = nullValue == null ? null : nullValue.toString();
}
@ -354,6 +355,9 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (similarity() != null) {
builder.field("similarity", similarity().name());
}
if (nullValue != null) {
builder.field("null_value", nullValue);
}

10
src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
View File

@ -45,6 +45,7 @@ import org.elasticsearch.index.field.data.FieldDataType;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
import org.elasticsearch.index.similarity.SimilarityProvider;
import java.io.IOException;
import java.util.Map;
@ -113,7 +114,7 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter,
precisionStep, fuzzyFactor, boost, fieldType, nullValue,
timeUnit, parseUpperInclusive, ignoreMalformed(context), provider);
timeUnit, parseUpperInclusive, ignoreMalformed(context), provider, similarity);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@ -152,12 +153,12 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter, int precisionStep, String fuzzyFactor,
float boost, FieldType fieldType,
String nullValue, TimeUnit timeUnit, boolean parseUpperInclusive, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider) {
PostingsFormatProvider provider, SimilarityProvider similarity) {
super(names, precisionStep, fuzzyFactor, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_date/" + precisionStep,
new NumericDateAnalyzer(precisionStep, dateTimeFormatter.parser())),
new NamedAnalyzer("_date/max", new NumericDateAnalyzer(Integer.MAX_VALUE, dateTimeFormatter.parser())),
provider);
provider, similarity);
this.dateTimeFormatter = dateTimeFormatter;
this.nullValue = nullValue;
this.timeUnit = timeUnit;
@ -428,6 +429,9 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
builder.field("fuzzy_factor", fuzzyFactor);
}
builder.field("format", dateTimeFormatter.format());
if (similarity() != null) {
builder.field("similarity", similarity().name());
}
if (nullValue != null) {
builder.field("null_value", nullValue);
}

10
src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
View File

@ -42,6 +42,7 @@ import org.elasticsearch.index.field.data.FieldDataType;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
import org.elasticsearch.index.similarity.SimilarityProvider;
import java.io.IOException;
import java.util.Map;
@ -86,7 +87,7 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
DoubleFieldMapper fieldMapper = new DoubleFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, boost, fieldType, nullValue,
ignoreMalformed(context), provider);
ignoreMalformed(context), provider, similarity);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@ -116,10 +117,10 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
protected DoubleFieldMapper(Names names, int precisionStep, String fuzzyFactor,
float boost, FieldType fieldType,
Double nullValue, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider) {
PostingsFormatProvider provider, SimilarityProvider similarity) {
super(names, precisionStep, fuzzyFactor, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_double/" + precisionStep, new NumericDoubleAnalyzer(precisionStep)),
new NamedAnalyzer("_double/max", new NumericDoubleAnalyzer(Integer.MAX_VALUE)), provider);
new NamedAnalyzer("_double/max", new NumericDoubleAnalyzer(Integer.MAX_VALUE)), provider, similarity);
this.nullValue = nullValue;
this.nullValueAsString = nullValue == null ? null : nullValue.toString();
}
@ -358,6 +359,9 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (similarity() != null) {
builder.field("similarity", similarity().name());
}
if (nullValue != null) {
builder.field("null_value", nullValue);
}

10
src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
View File

@ -43,6 +43,7 @@ import org.elasticsearch.index.field.data.FieldDataType;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
import org.elasticsearch.index.similarity.SimilarityProvider;
import java.io.IOException;
import java.util.Map;
@ -87,7 +88,7 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
FloatFieldMapper fieldMapper = new FloatFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, boost, fieldType, nullValue,
ignoreMalformed(context), provider);
ignoreMalformed(context), provider, similarity);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@ -114,10 +115,10 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
private String nullValueAsString;
protected FloatFieldMapper(Names names, int precisionStep, String fuzzyFactor, float boost, FieldType fieldType,
Float nullValue, Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider) {
Float nullValue, Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity) {
super(names, precisionStep, fuzzyFactor, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_float/" + precisionStep, new NumericFloatAnalyzer(precisionStep)),
new NamedAnalyzer("_float/max", new NumericFloatAnalyzer(Integer.MAX_VALUE)), provider);
new NamedAnalyzer("_float/max", new NumericFloatAnalyzer(Integer.MAX_VALUE)), provider, similarity);
this.nullValue = nullValue;
this.nullValueAsString = nullValue == null ? null : nullValue.toString();
}
@ -353,6 +354,9 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (similarity() != null) {
builder.field("similarity", similarity().name());
}
if (nullValue != null) {
builder.field("null_value", nullValue);
}

10
src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
View File

@ -43,6 +43,7 @@ import org.elasticsearch.index.field.data.FieldDataType;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
import org.elasticsearch.index.similarity.SimilarityProvider;
import java.io.IOException;
import java.util.Map;
@ -87,7 +88,7 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
IntegerFieldMapper fieldMapper = new IntegerFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, boost, fieldType,
nullValue, ignoreMalformed(context), provider);
nullValue, ignoreMalformed(context), provider, similarity);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@ -116,10 +117,10 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
protected IntegerFieldMapper(Names names, int precisionStep, String fuzzyFactor,
float boost, FieldType fieldType,
Integer nullValue, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider) {
PostingsFormatProvider provider, SimilarityProvider similarity) {
super(names, precisionStep, fuzzyFactor, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_int/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)),
new NamedAnalyzer("_int/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider);
new NamedAnalyzer("_int/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity);
this.nullValue = nullValue;
this.nullValueAsString = nullValue == null ? null : nullValue.toString();
}
@ -358,6 +359,9 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (similarity() != null) {
builder.field("similarity", similarity().name());
}
if (nullValue != null) {
builder.field("null_value", nullValue);
}

10
src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
View File

@ -43,6 +43,7 @@ import org.elasticsearch.index.field.data.FieldDataType;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
import org.elasticsearch.index.similarity.SimilarityProvider;
import java.io.IOException;
import java.util.Map;
@ -87,7 +88,7 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
fieldType.setOmitNorms(fieldType.omitNorms() && boost != 1.0f);
LongFieldMapper fieldMapper = new LongFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, boost, fieldType, nullValue,
ignoreMalformed(context), provider);
ignoreMalformed(context), provider, similarity);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@ -116,10 +117,10 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
protected LongFieldMapper(Names names, int precisionStep, String fuzzyFactor,
float boost, FieldType fieldType,
Long nullValue, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider) {
PostingsFormatProvider provider, SimilarityProvider similarity) {
super(names, precisionStep, fuzzyFactor, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_long/" + precisionStep, new NumericLongAnalyzer(precisionStep)),
new NamedAnalyzer("_long/max", new NumericLongAnalyzer(Integer.MAX_VALUE)), provider);
new NamedAnalyzer("_long/max", new NumericLongAnalyzer(Integer.MAX_VALUE)), provider, similarity);
this.nullValue = nullValue;
this.nullValueAsString = nullValue == null ? null : nullValue.toString();
}
@ -357,6 +358,9 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (similarity() != null) {
builder.field("similarity", similarity().name());
}
if (nullValue != null) {
builder.field("null_value", nullValue);
}

5
src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
View File

@ -37,6 +37,7 @@ import org.elasticsearch.index.field.data.FieldDataType;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.internal.AllFieldMapper;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.similarity.SimilarityProvider;
import java.io.IOException;
import java.io.Reader;
@ -141,9 +142,9 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
protected NumberFieldMapper(Names names, int precisionStep, @Nullable String fuzzyFactor,
float boost, FieldType fieldType,
Explicit<Boolean> ignoreMalformed, NamedAnalyzer indexAnalyzer,
NamedAnalyzer searchAnalyzer, PostingsFormatProvider provider) {
NamedAnalyzer searchAnalyzer, PostingsFormatProvider provider, SimilarityProvider similarity) {
// LUCENE 4 UPGRADE: Since we can't do anything before the super call, we have to push the boost check down to subclasses
super(names, boost, fieldType, indexAnalyzer, searchAnalyzer, provider);
super(names, boost, fieldType, indexAnalyzer, searchAnalyzer, provider, similarity);
if (precisionStep <= 0 || precisionStep >= maxPrecisionStep()) {
this.precisionStep = Integer.MAX_VALUE;
} else {

10
src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
View File

@ -43,6 +43,7 @@ import org.elasticsearch.index.field.data.FieldDataType;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
import org.elasticsearch.index.similarity.SimilarityProvider;
import java.io.IOException;
import java.util.Map;
@ -87,7 +88,7 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
ShortFieldMapper fieldMapper = new ShortFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, boost, fieldType, nullValue,
ignoreMalformed(context), provider);
ignoreMalformed(context), provider, similarity);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@ -116,10 +117,10 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
protected ShortFieldMapper(Names names, int precisionStep, String fuzzyFactor,
float boost, FieldType fieldType,
Short nullValue, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider) {
PostingsFormatProvider provider, SimilarityProvider similarity) {
super(names, precisionStep, fuzzyFactor, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_short/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)),
new NamedAnalyzer("_short/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider);
new NamedAnalyzer("_short/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity);
this.nullValue = nullValue;
this.nullValueAsString = nullValue == null ? null : nullValue.toString();
}
@ -357,6 +358,9 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (similarity() != null) {
builder.field("similarity", similarity().name());
}
if (nullValue != null) {
builder.field("null_value", nullValue);
}

14
src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
View File

@ -33,6 +33,7 @@ import org.elasticsearch.index.analysis.NamedCustomAnalyzer;
import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.internal.AllFieldMapper;
import org.elasticsearch.index.similarity.SimilarityProvider;
import java.io.IOException;
import java.util.Map;
@ -130,7 +131,7 @@ public class StringFieldMapper extends AbstractFieldMapper<String> implements Al
}
StringFieldMapper fieldMapper = new StringFieldMapper(buildNames(context),
boost, fieldType, nullValue, indexAnalyzer, searchAnalyzer, searchQuotedAnalyzer,
positionOffsetGap, ignoreAbove, provider);
positionOffsetGap, ignoreAbove, provider, similarity);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@ -185,16 +186,16 @@ public class StringFieldMapper extends AbstractFieldMapper<String> implements Al
protected StringFieldMapper(Names names, float boost, FieldType fieldType,
String nullValue, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer,
PostingsFormatProvider postingsFormat) {
PostingsFormatProvider postingsFormat, SimilarityProvider similarity) {
this(names, boost, fieldType, nullValue, indexAnalyzer, searchAnalyzer, searchAnalyzer,
Defaults.POSITION_OFFSET_GAP, Defaults.IGNORE_ABOVE, postingsFormat);
Defaults.POSITION_OFFSET_GAP, Defaults.IGNORE_ABOVE, postingsFormat, similarity);
}
protected StringFieldMapper(Names names, float boost, FieldType fieldType,
String nullValue, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer,
NamedAnalyzer searchQuotedAnalyzer, int positionOffsetGap, int ignoreAbove,
PostingsFormatProvider postingsFormat) {
super(names, boost, fieldType, indexAnalyzer, searchAnalyzer, postingsFormat);
PostingsFormatProvider postingsFormat, SimilarityProvider similarity) {
super(names, boost, fieldType, indexAnalyzer, searchAnalyzer, postingsFormat, similarity);
this.nullValue = nullValue;
this.positionOffsetGap = positionOffsetGap;
this.searchQuotedAnalyzer = searchQuotedAnalyzer != null ? searchQuotedAnalyzer : this.searchAnalyzer;
@ -361,6 +362,9 @@ public class StringFieldMapper extends AbstractFieldMapper<String> implements Al
if (searchQuotedAnalyzer != null && searchAnalyzer != searchQuotedAnalyzer) {
builder.field("search_quote_analyzer", searchQuotedAnalyzer.name());
}
if (similarity() != null) {
builder.field("similarity", similarity().name());
}
if (ignoreAbove != Defaults.IGNORE_ABOVE) {
builder.field("ignore_above", ignoreAbove);
}

4
src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java
View File

@ -55,6 +55,8 @@ public class TypeParsers {
builder.ignoreMalformed(nodeBooleanValue(propNode));
} else if (propName.equals("omit_norms")) {
builder.omitNorms(nodeBooleanValue(propNode));
} else if (propName.equals("similarity")) {
builder.similarity(parserContext.similarityLookupService().similarity(propNode.toString()));
}
}
}
@ -114,6 +116,8 @@ public class TypeParsers {
} else if (propName.equals("postings_format")) {
String postingFormatName = propNode.toString();
builder.postingsFormat(parserContext.postingFormatService().get(postingFormatName));
} else if (propName.equals("similarity")) {
builder.similarity(parserContext.similarityLookupService().similarity(propNode.toString()));
}
}
}

2
src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
View File

@ -572,7 +572,7 @@ public class GeoPointFieldMapper implements Mapper, ArrayValueMapperParser {
public GeoStringFieldMapper(Names names, float boost, FieldType fieldType, String nullValue,
NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer,
PostingsFormatProvider provider) {
super(names, boost, fieldType, nullValue, indexAnalyzer, searchAnalyzer, provider);
super(names, boost, fieldType, nullValue, indexAnalyzer, searchAnalyzer, provider, null);
}
@Override

2
src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java
View File

@ -138,7 +138,7 @@ public class GeoShapeFieldMapper extends AbstractFieldMapper<String> {
public GeoShapeFieldMapper(FieldMapper.Names names, SpatialPrefixTree prefixTree, double distanceErrorPct,
FieldType fieldType, PostingsFormatProvider provider) {
super(names, 1, fieldType, null, null, provider);
super(names, 1, fieldType, null, null, provider, null);
this.spatialStrategy = new TermQueryPrefixTreeStrategy(names, prefixTree, distanceErrorPct);
}

12
src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
View File

@ -36,6 +36,7 @@ import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.similarity.SimilarityProvider;
import java.io.IOException;
import java.util.Map;
@ -98,7 +99,7 @@ public class AllFieldMapper extends AbstractFieldMapper<Void> implements Interna
fieldType.setIndexed(true);
fieldType.setTokenized(true);
return new AllFieldMapper(name, fieldType, indexAnalyzer, searchAnalyzer, enabled, autoBoost, provider);
return new AllFieldMapper(name, fieldType, indexAnalyzer, searchAnalyzer, enabled, autoBoost, provider, similarity);
}
}
@ -130,12 +131,12 @@ public class AllFieldMapper extends AbstractFieldMapper<Void> implements Interna
private volatile boolean autoBoost;
public AllFieldMapper() {
this(Defaults.NAME, new FieldType(Defaults.ALL_FIELD_TYPE), null, null, Defaults.ENABLED, false, null);
this(Defaults.NAME, new FieldType(Defaults.ALL_FIELD_TYPE), null, null, Defaults.ENABLED, false, null, null);
}
protected AllFieldMapper(String name, FieldType fieldType, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer,
boolean enabled, boolean autoBoost, PostingsFormatProvider provider) {
super(new Names(name, name, name, name), 1.0f, fieldType, indexAnalyzer, searchAnalyzer, provider);
boolean enabled, boolean autoBoost, PostingsFormatProvider provider, SimilarityProvider similarity) {
super(new Names(name, name, name, name), 1.0f, fieldType, indexAnalyzer, searchAnalyzer, provider, similarity);
this.enabled = enabled;
this.autoBoost = autoBoost;
@ -285,6 +286,9 @@ public class AllFieldMapper extends AbstractFieldMapper<Void> implements Interna
builder.field("search_analyzer", searchAnalyzer.name());
}
}
if (similarity() != null) {
builder.field("similarity", similarity().name());
}
builder.endObject();
return builder;
}

2
src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java
View File

@ -122,7 +122,7 @@ public class BoostFieldMapper extends NumberFieldMapper<Float> implements Intern
Float nullValue, PostingsFormatProvider provider) {
super(new Names(name, indexName, indexName, name), precisionStep, null, boost, fieldType,
Defaults.IGNORE_MALFORMED, new NamedAnalyzer("_float/" + precisionStep, new NumericFloatAnalyzer(precisionStep)),
new NamedAnalyzer("_float/max", new NumericFloatAnalyzer(Integer.MAX_VALUE)), provider);
new NamedAnalyzer("_float/max", new NumericFloatAnalyzer(Integer.MAX_VALUE)), provider, null);
this.nullValue = nullValue;
}

2
src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
View File

@ -125,7 +125,7 @@ public class IdFieldMapper extends AbstractFieldMapper<String> implements Intern
protected IdFieldMapper(String name, String indexName, float boost, FieldType fieldType, String path,
PostingsFormatProvider provider) {
super(new Names(name, indexName, indexName, name), boost, fieldType, Lucene.KEYWORD_ANALYZER,
Lucene.KEYWORD_ANALYZER, provider);
Lucene.KEYWORD_ANALYZER, provider, null);
this.path = path;
}

2
src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
View File

@ -114,7 +114,7 @@ public class IndexFieldMapper extends AbstractFieldMapper<String> implements Int
public IndexFieldMapper(String name, String indexName, float boost, FieldType fieldType, boolean enabled,
PostingsFormatProvider provider) {
super(new Names(name, indexName, indexName, name), boost, fieldType, Lucene.KEYWORD_ANALYZER,
Lucene.KEYWORD_ANALYZER, provider);
Lucene.KEYWORD_ANALYZER, provider, null);
this.enabled = enabled;
}

2
src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
View File

@ -116,7 +116,7 @@ public class ParentFieldMapper extends AbstractFieldMapper<Uid> implements Inter
protected ParentFieldMapper(String name, String indexName, String type, PostingsFormatProvider postingsFormat) {
super(new Names(name, indexName, indexName, name), Defaults.BOOST, new FieldType(Defaults.PARENT_FIELD_TYPE),
Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, postingsFormat);
Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, postingsFormat, null);
this.type = type;
}

2
src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
View File

@ -119,7 +119,7 @@ public class RoutingFieldMapper extends AbstractFieldMapper<String> implements I
protected RoutingFieldMapper(FieldType fieldType, boolean required, String path, PostingsFormatProvider provider) {
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), 1.0f, fieldType, Lucene.KEYWORD_ANALYZER,
Lucene.KEYWORD_ANALYZER, provider);
Lucene.KEYWORD_ANALYZER, provider, null);
this.required = required;
this.path = path;
}

2
src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java
View File

@ -100,7 +100,7 @@ public class SizeFieldMapper extends IntegerFieldMapper implements RootMapper {
public SizeFieldMapper(boolean enabled, FieldType fieldType, PostingsFormatProvider provider) {
super(new Names(Defaults.NAME), Defaults.PRECISION_STEP, Defaults.FUZZY_FACTOR,
Defaults.BOOST, fieldType, Defaults.NULL_VALUE, Defaults.IGNORE_MALFORMED, provider);
Defaults.BOOST, fieldType, Defaults.NULL_VALUE, Defaults.IGNORE_MALFORMED, provider, null);
this.enabled = enabled;
}

2
src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
View File

@ -200,7 +200,7 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements In
protected SourceFieldMapper(String name, boolean enabled, String format, Boolean compress, long compressThreshold,
String[] includes, String[] excludes) {
super(new Names(name, name, name, name), Defaults.BOOST, new FieldType(Defaults.SOURCE_FIELD_TYPE),
Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, null); // Only stored.
Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, null, null); // Only stored.
this.enabled = enabled;
this.compress = compress;
this.compressThreshold = compressThreshold;

2
src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
View File

@ -119,7 +119,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R
PostingsFormatProvider provider) {
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), Defaults.PRECISION_STEP,
Defaults.FUZZY_FACTOR, Defaults.BOOST, fieldType, Defaults.NULL_VALUE, ignoreMalformed,
provider);
provider, null);
this.enabled = enabled;
this.defaultTTL = defaultTTL;
}

2
src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
View File

@ -138,7 +138,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter,
Defaults.PRECISION_STEP, Defaults.FUZZY_FACTOR, Defaults.BOOST, fieldType,
Defaults.NULL_VALUE, TimeUnit.MILLISECONDS /*always milliseconds*/,
parseUpperInclusive, ignoreMalformed, provider);
parseUpperInclusive, ignoreMalformed, provider, null);
this.enabled = enabled;
this.path = path;
}

2
src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
View File

@ -103,7 +103,7 @@ public class TypeFieldMapper extends AbstractFieldMapper<String> implements Inte
public TypeFieldMapper(String name, String indexName, float boost, FieldType fieldType, PostingsFormatProvider provider) {
super(new Names(name, indexName, indexName, name), boost, fieldType, Lucene.KEYWORD_ANALYZER,
Lucene.KEYWORD_ANALYZER, provider);
Lucene.KEYWORD_ANALYZER, provider, null);
}
public String value(Document document) {

2
src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
View File

@ -109,7 +109,7 @@ public class UidFieldMapper extends AbstractFieldMapper<Uid> implements Internal
protected UidFieldMapper(String name, String indexName, PostingsFormatProvider postingsFormat) {
super(new Names(name, indexName, indexName, name), Defaults.BOOST, new FieldType(Defaults.UID_FIELD_TYPE),
Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, postingsFormat);
Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, postingsFormat, null);
}
@Override

10
src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
View File

@ -43,6 +43,7 @@ import org.elasticsearch.index.mapper.core.LongFieldMapper;
import org.elasticsearch.index.mapper.core.NumberFieldMapper;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
import org.elasticsearch.index.similarity.SimilarityProvider;
import java.io.IOException;
import java.io.Reader;
@ -113,7 +114,7 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
public IpFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
IpFieldMapper fieldMapper = new IpFieldMapper(buildNames(context),
precisionStep, boost, fieldType, nullValue, ignoreMalformed(context), provider);
precisionStep, boost, fieldType, nullValue, ignoreMalformed(context), provider, similarity);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@ -140,10 +141,10 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
protected IpFieldMapper(Names names, int precisionStep,
float boost, FieldType fieldType,
String nullValue, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider) {
PostingsFormatProvider provider, SimilarityProvider similarity) {
super(names, precisionStep, null, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)),
new NamedAnalyzer("_ip/max", new NumericIpAnalyzer(Integer.MAX_VALUE)), provider);
new NamedAnalyzer("_ip/max", new NumericIpAnalyzer(Integer.MAX_VALUE)), provider, similarity);
this.nullValue = nullValue;
}
@ -331,6 +332,9 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
if (precisionStep != Defaults.PRECISION_STEP) {
builder.field("precision_step", precisionStep);
}
if (similarity() != null) {
builder.field("similarity", similarity().name());
}
if (nullValue != null) {
builder.field("null_value", nullValue);
}

2
src/main/java/org/elasticsearch/index/query/QueryParseContext.java
View File

@ -123,7 +123,7 @@ public class QueryParseContext {
}
public Similarity searchSimilarity() {
return indexQueryParser.similarityService != null ? indexQueryParser.similarityService.defaultSearchSimilarity() : null;
return indexQueryParser.similarityService != null ? indexQueryParser.similarityService.similarity() : null;
}
public IndexCache indexCache() {

51
src/main/java/org/elasticsearch/index/similarity/AbstractSimilarityProvider.java
View File

@ -19,26 +19,61 @@
package org.elasticsearch.index.similarity;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.*;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.settings.IndexSettings;
/**
*
* Abstract implemenetation of {@link SimilarityProvider} providing common behaviour
*/
public abstract class AbstractSimilarityProvider<T extends Similarity> extends AbstractIndexComponent implements SimilarityProvider<T> {
public abstract class AbstractSimilarityProvider implements SimilarityProvider {
protected static final Normalization NO_NORMALIZATION = new Normalization.NoNormalization();
private final String name;
protected AbstractSimilarityProvider(Index index, @IndexSettings Settings indexSettings, String name) {
super(index, indexSettings);
/**
* Creates a new AbstractSimilarityProvider with the given name
*
* @param name Name of the Provider
*/
protected AbstractSimilarityProvider(String name) {
this.name = name;
}
/**
* {@inheritDoc}
*/
@Override
public String name() {
return this.name;
}
/**
* Parses the given Settings and creates the appropriate {@link Normalization}
*
* @param settings Settings to parse
* @return {@link Normalization} referred to in the Settings
*/
protected Normalization parseNormalization(Settings settings) {
String normalization = settings.get("normalization");
if ("no".equals(normalization)) {
return NO_NORMALIZATION;
} else if ("h1".equals(normalization)) {
float c = settings.getAsFloat("normalization.h1.c", 1f);
return new NormalizationH1(c);
} else if ("h2".equals(normalization)) {
float c = settings.getAsFloat("normalization.h2.c", 1f);
return new NormalizationH2(c);
} else if ("h3".equals(normalization)) {
float c = settings.getAsFloat("normalization.h3.c", 800f);
return new NormalizationH3(c);
} else if ("z".equals(normalization)) {
float z = settings.getAsFloat("normalization.z.z", 0.30f);
return new NormalizationZ(z);
} else {
throw new ElasticSearchIllegalArgumentException("Unsupported Normalization [" + normalization + "]");
}
}
}

61
src/main/java/org/elasticsearch/index/similarity/BM25SimilarityProvider.java Normal file
View File

@ -0,0 +1,61 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.similarity;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.Similarity;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.assistedinject.Assisted;
import org.elasticsearch.common.settings.Settings;
/**
* {@link SimilarityProvider} for the {@link BM25Similarity}.
* <p/>
* Configuration options available:
* <ul>
* <li>k1</li>
* <li>b</li>
* <li>discount_overlaps</li>
* </ul>
* @see BM25Similarity For more information about configuration
*/
public class BM25SimilarityProvider extends AbstractSimilarityProvider {
private final BM25Similarity similarity;
@Inject
public BM25SimilarityProvider(@Assisted String name, @Assisted Settings settings) {
super(name);
float k1 = settings.getAsFloat("k1", 1.2f);
float b = settings.getAsFloat("b", 0.75f);
boolean discountOverlaps = settings.getAsBoolean("discount_overlaps", true);
this.similarity = new BM25Similarity(k1, b);
this.similarity.setDiscountOverlaps(discountOverlaps);
}
/**
* {@inheritDoc}
*/
@Override
public Similarity get() {
return similarity;
}
}

112
src/main/java/org/elasticsearch/index/similarity/DFRSimilarityProvider.java Normal file
View File

@ -0,0 +1,112 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.similarity;
import com.google.common.collect.ImmutableMap;
import org.apache.lucene.search.similarities.*;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.assistedinject.Assisted;
import org.elasticsearch.common.settings.Settings;
/**
* {@link SimilarityProvider} for {@link DFRSimilarity}.
* <p/>
* Configuration options available:
* <ul>
* <li>basic_model</li>
* <li>after_effect</li>
* <li>normalization</li>
* </ul>
* @see DFRSimilarity For more information about configuration
*/
public class DFRSimilarityProvider extends AbstractSimilarityProvider {
private static final ImmutableMap<String, BasicModel> MODEL_CACHE;
private static final ImmutableMap<String, AfterEffect> EFFECT_CACHE;
static {
MapBuilder<String, BasicModel> models = MapBuilder.newMapBuilder();
models.put("be", new BasicModelBE());
models.put("d", new BasicModelD());
models.put("g", new BasicModelG());
models.put("if", new BasicModelIF());
models.put("in", new BasicModelIn());
models.put("ine", new BasicModelIne());
models.put("p", new BasicModelP());
MODEL_CACHE = models.immutableMap();
MapBuilder<String, AfterEffect> effects = MapBuilder.newMapBuilder();
effects.put("no", new AfterEffect.NoAfterEffect());
effects.put("b", new AfterEffectB());
effects.put("l", new AfterEffectL());
EFFECT_CACHE = effects.immutableMap();
}
private final DFRSimilarity similarity;
@Inject
public DFRSimilarityProvider(@Assisted String name, @Assisted Settings settings) {
super(name);
BasicModel basicModel = parseBasicModel(settings);
AfterEffect afterEffect = parseAfterEffect(settings);
Normalization normalization = parseNormalization(settings);
this.similarity = new DFRSimilarity(basicModel, afterEffect, normalization);
}
/**
* Parses the given Settings and creates the appropriate {@link BasicModel}
*
* @param settings Settings to parse
* @return {@link BasicModel} referred to in the Settings
*/
protected BasicModel parseBasicModel(Settings settings) {
String basicModel = settings.get("basic_model");
BasicModel model = MODEL_CACHE.get(basicModel);
if (model == null) {
throw new ElasticSearchIllegalArgumentException("Unsupported BasicModel [" + basicModel + "]");
}
return model;
}
/**
* Parses the given Settings and creates the appropriate {@link AfterEffect}
*
* @param settings Settings to parse
* @return {@link AfterEffect} referred to in the Settings
*/
protected AfterEffect parseAfterEffect(Settings settings) {
String afterEffect = settings.get("after_effect");
AfterEffect effect = EFFECT_CACHE.get(afterEffect);
if (effect == null) {
throw new ElasticSearchIllegalArgumentException("Unsupported AfterEffect [" + afterEffect + "]");
}
return effect;
}
/**
* {@inheritDoc}
*/
@Override
public Similarity get() {
return similarity;
}
}

22
src/main/java/org/elasticsearch/index/similarity/DefaultSimilarityProvider.java
View File

@ -27,18 +27,28 @@ import org.elasticsearch.index.Index;
import org.elasticsearch.index.settings.IndexSettings;
/**
*
* {@link SimilarityProvider} for {@link DefaultSimilarity}.
* <p/>
* Configuration options available:
* <ul>
* <li>discount_overlaps</li>
* </ul>
* @see DefaultSimilarity For more information about configuration
*/
public class DefaultSimilarityProvider extends AbstractSimilarityProvider<DefaultSimilarity> {
public class DefaultSimilarityProvider extends AbstractSimilarityProvider {
private DefaultSimilarity similarity;
private final DefaultSimilarity similarity = new DefaultSimilarity();
@Inject
public DefaultSimilarityProvider(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) {
super(index, indexSettings, name);
this.similarity = new DefaultSimilarity();
public DefaultSimilarityProvider(@Assisted String name, @Assisted Settings settings) {
super(name);
boolean discountOverlaps = settings.getAsBoolean("discount_overlaps", true);
this.similarity.setDiscountOverlaps(discountOverlaps);
}
/**
* {@inheritDoc}
*/
@Override
public DefaultSimilarity get() {
return similarity;

106
src/main/java/org/elasticsearch/index/similarity/IBSimilarityProvider.java Normal file
View File

@ -0,0 +1,106 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.similarity;
import com.google.common.collect.ImmutableMap;
import org.apache.lucene.search.similarities.*;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.assistedinject.Assisted;
import org.elasticsearch.common.settings.Settings;
/**
* {@link SimilarityProvider} for {@link IBSimilarity}.
* <p/>
* Configuration options available:
* <ul>
* <li>distribution</li>
* <li>lambda</li>
* <li>normalization</li>
* </ul>
* @see IBSimilarity For more information about configuration
*/
public class IBSimilarityProvider extends AbstractSimilarityProvider {
private static final ImmutableMap<String, Distribution> DISTRIBUTION_CACHE;
private static final ImmutableMap<String, Lambda> LAMBDA_CACHE;
static {
MapBuilder<String, Distribution> distributions = MapBuilder.newMapBuilder();
distributions.put("ll", new DistributionLL());
distributions.put("spl", new DistributionSPL());
DISTRIBUTION_CACHE = distributions.immutableMap();
MapBuilder<String, Lambda> lamdas = MapBuilder.newMapBuilder();
lamdas.put("df", new LambdaDF());
lamdas.put("ttf", new LambdaTTF());
LAMBDA_CACHE = lamdas.immutableMap();
}
private final IBSimilarity similarity;
@Inject
public IBSimilarityProvider(@Assisted String name, @Assisted Settings settings) {
super(name);
Distribution distribution = parseDistribution(settings);
Lambda lambda = parseLambda(settings);
Normalization normalization = parseNormalization(settings);
this.similarity = new IBSimilarity(distribution, lambda, normalization);
}
/**
* Parses the given Settings and creates the appropriate {@link Distribution}
*
* @param settings Settings to parse
* @return {@link Normalization} referred to in the Settings
*/
protected Distribution parseDistribution(Settings settings) {
String rawDistribution = settings.get("distribution");
Distribution distribution = DISTRIBUTION_CACHE.get(rawDistribution);
if (distribution == null) {
throw new ElasticSearchIllegalArgumentException("Unsupported Distribution [" + rawDistribution + "]");
}
return distribution;
}
/**
* Parses the given Settings and creates the appropriate {@link Lambda}
*
* @param settings Settings to parse
* @return {@link Normalization} referred to in the Settings
*/
protected Lambda parseLambda(Settings settings) {
String rawLambda = settings.get("lambda");
Lambda lambda = LAMBDA_CACHE.get(rawLambda);
if (lambda == null) {
throw new ElasticSearchIllegalArgumentException("Unsupported Lambda [" + rawLambda + "]");
}
return lambda;
}
/**
* {@inheritDoc}
*/
@Override
public Similarity get() {
return similarity;
}
}

73
src/main/java/org/elasticsearch/index/similarity/PreBuiltSimilarityProvider.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.similarity;
import org.apache.lucene.search.similarities.Similarity;
import org.elasticsearch.common.settings.Settings;
/**
* {@link SimilarityProvider} for pre-built Similarities
*/
public class PreBuiltSimilarityProvider extends AbstractSimilarityProvider {
public static class Factory implements SimilarityProvider.Factory {
private final PreBuiltSimilarityProvider similarity;
public Factory(String name, Similarity similarity) {
this.similarity = new PreBuiltSimilarityProvider(name, similarity);
}
@Override
public SimilarityProvider create(String name, Settings settings) {
return similarity;
}
public String name() {
return similarity.name();
}
public SimilarityProvider get() {
return similarity;
}
}
private final Similarity similarity;
/**
* Creates a new {@link PreBuiltSimilarityProvider} with the given name and given
* pre-built Similarity
*
* @param name Name of the Provider
* @param similarity Pre-built Similarity
*/
public PreBuiltSimilarityProvider(String name, Similarity similarity) {
super(name);
this.similarity = similarity;
}
/**
* {@inheritDoc}
*/
@Override
public Similarity get() {
return similarity;
}
}

53
src/main/java/org/elasticsearch/index/similarity/Similarities.java Normal file
View File

@ -0,0 +1,53 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.similarity;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableMap;
import org.apache.lucene.search.similarities.*;
import org.elasticsearch.common.collect.MapBuilder;
/**
* Cache of pre-defined Similarities
*/
public class Similarities {
private static final ImmutableMap<String, PreBuiltSimilarityProvider.Factory> PRE_BUILT_SIMILARITIES;
static {
MapBuilder<String, PreBuiltSimilarityProvider.Factory> similarities = MapBuilder.newMapBuilder();
similarities.put("default", new PreBuiltSimilarityProvider.Factory("default", new DefaultSimilarity()));
similarities.put("BM25", new PreBuiltSimilarityProvider.Factory("BM25", new BM25Similarity()));
PRE_BUILT_SIMILARITIES = similarities.immutableMap();
}
private Similarities() {
}
/**
* Returns the list of pre-defined SimilarityProvider Factories
*
* @return Pre-defined SimilarityProvider Factories
*/
public static ImmutableCollection<PreBuiltSimilarityProvider.Factory> listFactories() {
return PRE_BUILT_SIMILARITIES.values();
}
}

84
src/main/java/org/elasticsearch/index/similarity/SimilarityLookupService.java Normal file
View File

@ -0,0 +1,84 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.similarity;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.settings.IndexSettings;
import java.util.Map;
/**
* Service for looking up configured {@link SimilarityProvider} implementations by name.
*
* The service instantiates the Providers through their Factories using configuration
* values found with the {@link SimilarityModule#SIMILARITY_SETTINGS_PREFIX} prefix.
*/
public class SimilarityLookupService extends AbstractIndexComponent {
private final ImmutableMap<String, SimilarityProvider> similarities;
public SimilarityLookupService(Index index, Settings indexSettings) {
this (index, indexSettings, ImmutableMap.<String, SimilarityProvider.Factory>of());
}
@Inject
public SimilarityLookupService(Index index, @IndexSettings Settings indexSettings, Map<String, SimilarityProvider.Factory> similarities) {
super(index, indexSettings);
MapBuilder<String, SimilarityProvider> providers = MapBuilder.newMapBuilder();
Map<String, Settings> similaritySettings = indexSettings.getGroups(SimilarityModule.SIMILARITY_SETTINGS_PREFIX);
for (Map.Entry<String, SimilarityProvider.Factory> entry : similarities.entrySet()) {
String name = entry.getKey();
SimilarityProvider.Factory factory = entry.getValue();
Settings settings = similaritySettings.get(name);
if (settings == null) {
settings = ImmutableSettings.Builder.EMPTY_SETTINGS;
}
providers.put(name, factory.create(name, settings));
}
// For testing
for (PreBuiltSimilarityProvider.Factory factory : Similarities.listFactories()) {
if (!providers.containsKey(factory.name())) {
providers.put(factory.name(), factory.get());
}
}
this.similarities = providers.immutableMap();
}
/**
* Returns the {@link SimilarityProvider} with the given name
*
* @param name Name of the SimilarityProvider to find
* @return {@link SimilarityProvider} with the given name, or {@code null} if no Provider exists
*/
public SimilarityProvider similarity(String name) {
return similarities.get(name);
}
}

51
src/main/java/org/elasticsearch/index/similarity/SimilarityModule.java
View File

@ -19,6 +19,8 @@
package org.elasticsearch.index.similarity;
import com.google.common.collect.Maps;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Scopes;
import org.elasticsearch.common.inject.assistedinject.FactoryProvider;
@ -28,33 +30,66 @@ import org.elasticsearch.common.settings.Settings;
import java.util.Map;
/**
* {@link SimilarityModule} is responsible gathering registered and configured {@link SimilarityProvider}
* implementations and making them available through the {@link SimilarityLookupService} and {@link SimilarityService}.
*
* New {@link SimilarityProvider} implementations can be registered through {@link #addSimilarity(String, Class)}
* while existing Providers can be referenced through Settings under the {@link #SIMILARITY_SETTINGS_PREFIX} prefix
* along with the "type" value. For example, to reference the {@link BM25SimilarityProvider}, the configuration
* <tt>"index.similarity.my_similarity.type : "BM25"</tt> can be used.
*/
public class SimilarityModule extends AbstractModule {
public static final String SIMILARITY_SETTINGS_PREFIX = "index.similarity";
private final Settings settings;
private final Map<String, Class<? extends SimilarityProvider>> similarities = Maps.newHashMap();
public SimilarityModule(Settings settings) {
this.settings = settings;
}
/**
* Registers the given {@link SimilarityProvider} with the given name
*
* @param name Name of the SimilarityProvider
* @param similarity SimilarityProvider to register
*/
public void addSimilarity(String name, Class<? extends SimilarityProvider> similarity) {
similarities.put(name, similarity);
}
@Override
protected void configure() {
MapBinder<String, SimilarityProviderFactory> similarityBinder
= MapBinder.newMapBinder(binder(), String.class, SimilarityProviderFactory.class);
Map<String, Class<? extends SimilarityProvider>> providers = Maps.newHashMap(similarities);
Map<String, Settings> similarityProvidersSettings = settings.getGroups("index.similarity");
for (Map.Entry<String, Settings> entry : similarityProvidersSettings.entrySet()) {
Map<String, Settings> similaritySettings = settings.getGroups(SIMILARITY_SETTINGS_PREFIX);
for (Map.Entry<String, Settings> entry : similaritySettings.entrySet()) {
String name = entry.getKey();
Settings settings = entry.getValue();
Class<? extends SimilarityProvider> type = settings.getAsClass("type", null, "org.elasticsearch.index.similarity.", "SimilarityProvider");
Class<? extends SimilarityProvider> type =
settings.getAsClass("type", null, "org.elasticsearch.index.similarity.", "SimilarityProvider");
if (type == null) {
throw new IllegalArgumentException("Similarity [" + name + "] must have a type associated with it");
throw new ElasticSearchIllegalArgumentException("SimilarityProvider [" + name + "] must have an associated type");
}
similarityBinder.addBinding(name).toProvider(FactoryProvider.newFactory(SimilarityProviderFactory.class, type)).in(Scopes.SINGLETON);
providers.put(name, type);
}
bind(SimilarityService.class).in(Scopes.SINGLETON);
MapBinder<String, SimilarityProvider.Factory> similarityBinder =
MapBinder.newMapBinder(binder(), String.class, SimilarityProvider.Factory.class);
for (Map.Entry<String, Class<? extends SimilarityProvider>> entry : providers.entrySet()) {
similarityBinder.addBinding(entry.getKey()).toProvider(FactoryProvider.newFactory(SimilarityProvider.Factory.class, entry.getValue())).in(Scopes.SINGLETON);
}
for (PreBuiltSimilarityProvider.Factory factory : Similarities.listFactories()) {
if (!providers.containsKey(factory.name())) {
similarityBinder.addBinding(factory.name()).toInstance(factory);
}
}
bind(SimilarityLookupService.class).asEagerSingleton();
bind(SimilarityService.class).asEagerSingleton();
}
}

32
src/main/java/org/elasticsearch/index/similarity/SimilarityProvider.java
View File

@ -21,14 +21,40 @@ package org.elasticsearch.index.similarity;
import org.apache.lucene.search.similarities.Similarity;
import org.elasticsearch.common.inject.Provider;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexComponent;
/**
*
* Provider for {@link Similarity} instances
*/
public interface SimilarityProvider<T extends Similarity> extends IndexComponent, Provider<T> {
public interface SimilarityProvider {
/**
* Returns the name associated with the Provider
*
* @return Name of the Provider
*/
String name();
T get();
/**
* Returns the {@link Similarity} the Provider is for
*
* @return Provided {@link Similarity}
*/
Similarity get();
/**
* Factory for creating {@link SimilarityProvider} instances
*/
public static interface Factory {
/**
* Creates a new {@link SimilarityProvider} instance
*
* @param name Name of the provider
* @param settings Settings to be used by the Provider
* @return {@link SimilarityProvider} instance created by the Factory
*/
SimilarityProvider create(String name, Settings settings);
}
}

30
src/main/java/org/elasticsearch/index/similarity/SimilarityProviderFactory.java
View File

@ -1,30 +0,0 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.similarity;
import org.elasticsearch.common.settings.Settings;
/**
*
*/
public interface SimilarityProviderFactory {
SimilarityProvider create(String name, Settings settings);
}

103
src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
View File

@ -19,81 +19,90 @@
package org.elasticsearch.index.similarity;
import com.google.common.collect.ImmutableMap;
import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
import org.apache.lucene.search.similarities.Similarity;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.name.Named;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.settings.IndexSettings;
import java.util.Map;
import static com.google.common.collect.Maps.newHashMap;
/**
*
*/
public class SimilarityService extends AbstractIndexComponent {
private final ImmutableMap<String, SimilarityProvider> similarityProviders;
private final SimilarityLookupService similarityLookupService;
private final MapperService mapperService;
private final ImmutableMap<String, Similarity> similarities;
private final Similarity perFieldSimilarity;
public SimilarityService(Index index) {
this(index, ImmutableSettings.Builder.EMPTY_SETTINGS, null);
this(index, ImmutableSettings.Builder.EMPTY_SETTINGS);
}
public SimilarityService(Index index, Settings settings) {
this (index, settings, new SimilarityLookupService(index, settings), null);
}
@Inject
public SimilarityService(Index index, @IndexSettings Settings indexSettings,
@Nullable Map<String, SimilarityProviderFactory> providerFactories) {
final SimilarityLookupService similarityLookupService, final MapperService mapperService) {
super(index, indexSettings);
this.similarityLookupService = similarityLookupService;
this.mapperService = mapperService;
Map<String, SimilarityProvider> similarityProviders = newHashMap();
if (providerFactories != null) {
Map<String, Settings> providersSettings = indexSettings.getGroups("index.similarity");
for (Map.Entry<String, SimilarityProviderFactory> entry : providerFactories.entrySet()) {
String similarityName = entry.getKey();
SimilarityProviderFactory similarityProviderFactory = entry.getValue();
Similarity defaultSimilarity = similarityLookupService.similarity("default").get();
// Expert users can configure the base type as being different to default, but out-of-box we use default.
Similarity baseSimilarity = (similarityLookupService.similarity("base") != null) ? similarityLookupService.similarity("base").get() :
defaultSimilarity;
Settings similaritySettings = providersSettings.get(similarityName);
if (similaritySettings == null) {
similaritySettings = ImmutableSettings.Builder.EMPTY_SETTINGS;
}
SimilarityProvider similarityProvider = similarityProviderFactory.create(similarityName, similaritySettings);
similarityProviders.put(similarityName, similarityProvider);
}
}
// add defaults
if (!similarityProviders.containsKey("index")) {
similarityProviders.put("index", new DefaultSimilarityProvider(index, indexSettings, "index", ImmutableSettings.Builder.EMPTY_SETTINGS));
}
if (!similarityProviders.containsKey("search")) {
similarityProviders.put("search", new DefaultSimilarityProvider(index, indexSettings, "search", ImmutableSettings.Builder.EMPTY_SETTINGS));
}
this.similarityProviders = ImmutableMap.copyOf(similarityProviders);
Map<String, Similarity> similarities = newHashMap();
for (SimilarityProvider provider : similarityProviders.values()) {
similarities.put(provider.name(), provider.get());
}
this.similarities = ImmutableMap.copyOf(similarities);
this.perFieldSimilarity = (mapperService != null) ? new PerFieldSimilarity(defaultSimilarity, baseSimilarity, mapperService) :
defaultSimilarity;
}
public Similarity similarity(String name) {
return similarities.get(name);
public Similarity similarity() {
return perFieldSimilarity;
}
public Similarity defaultIndexSimilarity() {
return similarities.get("index");
public SimilarityLookupService similarityLookupService() {
return similarityLookupService;
}
public Similarity defaultSearchSimilarity() {
return similarities.get("search");
public MapperService mapperService() {
return mapperService;
}
static class PerFieldSimilarity extends PerFieldSimilarityWrapper {
private final Similarity defaultSimilarity;
private final Similarity baseSimilarity;
private final MapperService mapperService;
PerFieldSimilarity(Similarity defaultSimilarity, Similarity baseSimilarity, MapperService mapperService) {
this.defaultSimilarity = defaultSimilarity;
this.baseSimilarity = baseSimilarity;
this.mapperService = mapperService;
}
@Override
public float coord(int overlap, int maxOverlap) {
return baseSimilarity.coord(overlap, maxOverlap);
}
@Override
public float queryNorm(float valueForNormalization) {
return baseSimilarity.queryNorm(valueForNormalization);
}
@Override
public Similarity get(String name) {
FieldMapper mapper = mapperService.smartNameFieldMapper(name);
return (mapper != null && mapper.similarity() != null) ? mapper.similarity().get() : defaultSimilarity;
}
}
}

4
src/main/java/org/elasticsearch/search/SearchService.java
View File

@ -282,7 +282,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
SearchContext context = findContext(request.id());
contextProcessing(context);
try {
context.searcher().dfSource(new CachedDfSource(context.searcher().getIndexReader(), request.dfs(), context.similarityService().defaultSearchSimilarity()));
context.searcher().dfSource(new CachedDfSource(context.searcher().getIndexReader(), request.dfs(), context.similarityService().similarity()));
} catch (IOException e) {
freeContext(context);
cleanContext(context);
@ -348,7 +348,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
SearchContext context = findContext(request.id());
contextProcessing(context);
try {
context.searcher().dfSource(new CachedDfSource(context.searcher().getIndexReader(), request.dfs(), context.similarityService().defaultSearchSimilarity()));
context.searcher().dfSource(new CachedDfSource(context.searcher().getIndexReader(), request.dfs(), context.similarityService().similarity()));
} catch (IOException e) {
freeContext(context);
cleanContext(context);

1
src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
View File

@ -61,6 +61,7 @@ public class ContextIndexSearcher extends IndexSearcher {
super(searcher.reader());
this.searchContext = searchContext;
this.reader = searcher.reader();
setSimilarity(searcher.searcher().getSimilarity());
}
public void dfSource(CachedDfSource dfSource) {

103
src/test/java/org/elasticsearch/test/integration/similarity/SimilarityTests.java Normal file
View File

@ -0,0 +1,103 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.test.integration.similarity;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.test.integration.AbstractNodesTests;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.FilterBuilders.numericRangeFilter;
import static org.elasticsearch.index.query.QueryBuilders.filteredQuery;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.index.query.QueryBuilders.queryString;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;
public class SimilarityTests extends AbstractNodesTests {
private Client client;
@BeforeClass
public void createNodes() throws Exception {
startNode("node1");
client = getClient();
}
@AfterClass
public void closeNodes() {
client.close();
closeAllNodes();
}
protected Client getClient() {
return client("node1");
}
@Test
public void testCustomBM25Similarity() throws Exception {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (Exception e) {
// ignore
}
client.admin().indices().prepareCreate("test")
.addMapping("type1", jsonBuilder().startObject()
.startObject("type1")
.startObject("properties")
.startObject("field1")
.field("similarity", "custom")
.field("type", "string")
.endObject()
.startObject("field2")
.field("similarity", "default")
.field("type", "string")
.endObject()
.endObject()
.endObject())
.setSettings(ImmutableSettings.settingsBuilder()
.put("number_of_shards", 1)
.put("number_of_replicas", 0)
.put("similarity.custom.type", "BM25")
.put("similarity.custom.k1", 2.0f)
.put("similarity.custom.b", 1.5f)
).execute().actionGet();
client.prepareIndex("test", "type1", "1").setSource("field1", "the quick brown fox jumped over the lazy dog",
"field2", "the quick brown fox jumped over the lazy dog")
.setRefresh(true).execute().actionGet();
SearchResponse bm25SearchResponse = client.prepareSearch().setQuery(matchQuery("field1", "quick brown fox")).execute().actionGet();
assertThat(bm25SearchResponse.hits().totalHits(), equalTo(1l));
float bm25Score = bm25SearchResponse.hits().hits()[0].score();
SearchResponse defaultSearchResponse = client.prepareSearch().setQuery(matchQuery("field2", "quick brown fox")).execute().actionGet();
assertThat(defaultSearchResponse.hits().totalHits(), equalTo(1l));
float defaultScore = defaultSearchResponse.hits().hits()[0].score();
assertThat(bm25Score, not(equalTo(defaultScore)));
}
}

2
src/test/java/org/elasticsearch/test/unit/index/codec/CodecTests.java
View File

@ -44,6 +44,7 @@ import org.elasticsearch.index.codec.postingsformat.*;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MapperServiceModule;
import org.elasticsearch.index.settings.IndexSettingsModule;
import org.elasticsearch.index.similarity.SimilarityModule;
import org.testng.annotations.Test;
import static org.hamcrest.MatcherAssert.assertThat;
@ -245,6 +246,7 @@ public class CodecTests {
.add(new SettingsModule(settings))
.add(new IndexNameModule(index))
.add(new IndexSettingsModule(index, settings))
.add(new SimilarityModule(settings))
.add(new CodecModule(settings))
.add(new MapperServiceModule())
.add(new AnalysisModule(settings))

1
src/test/java/org/elasticsearch/test/unit/index/engine/robin/SimpleRobinEngineTests.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.test.unit.index.engine.robin;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.codec.CodecService;
import org.elasticsearch.index.engine.Engine;

18
src/test/java/org/elasticsearch/test/unit/index/mapper/MapperTests.java
View File

@ -19,6 +19,8 @@
package org.elasticsearch.test.unit.index.mapper;
import com.google.common.collect.Maps;
import org.apache.lucene.search.similarities.Similarity;
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.common.inject.ModulesBuilder;
import org.elasticsearch.common.settings.ImmutableSettings;
@ -34,25 +36,31 @@ import org.elasticsearch.index.codec.postingsformat.PostingsFormatService;
import org.elasticsearch.index.mapper.DocumentMapperParser;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.settings.IndexSettingsModule;
import org.elasticsearch.index.similarity.SimilarityLookupService;
import org.elasticsearch.indices.analysis.IndicesAnalysisModule;
import org.elasticsearch.indices.analysis.IndicesAnalysisService;
import java.util.HashMap;
import java.util.Map;
/**
*
*/
public class MapperTests {
public static DocumentMapperParser newParser() {
return new DocumentMapperParser(new Index("test"), newAnalysisService(), new PostingsFormatService(new Index("test")));
return new DocumentMapperParser(new Index("test"), newAnalysisService(), new PostingsFormatService(new Index("test")),
newSimilarityLookupService());
}
public static DocumentMapperParser newParser(Settings indexSettings) {
return new DocumentMapperParser(new Index("test"), indexSettings, newAnalysisService(), new PostingsFormatService(new Index("test")));
return new DocumentMapperParser(new Index("test"), indexSettings, newAnalysisService(), new PostingsFormatService(new Index("test")),
newSimilarityLookupService());
}
public static MapperService newMapperService() {
return new MapperService(new Index("test"), ImmutableSettings.Builder.EMPTY_SETTINGS, new Environment(), newAnalysisService(),
new PostingsFormatService(new Index("test")));
new PostingsFormatService(new Index("test")), newSimilarityLookupService());
}
public static AnalysisService newAnalysisService() {
@ -64,4 +72,8 @@ public class MapperTests {
return injector.getInstance(AnalysisService.class);
}
public static SimilarityLookupService newSimilarityLookupService() {
return new SimilarityLookupService(new Index("test"), ImmutableSettings.Builder.EMPTY_SETTINGS);
}
}

169
src/test/java/org/elasticsearch/test/unit/index/similarity/SimilarityTests.java Normal file
View File

@ -0,0 +1,169 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.test.unit.index.similarity;
import org.apache.lucene.search.similarities.*;
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.common.inject.ModulesBuilder;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsModule;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexNameModule;
import org.elasticsearch.index.analysis.AnalysisModule;
import org.elasticsearch.index.codec.CodecModule;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MapperServiceModule;
import org.elasticsearch.index.settings.IndexSettingsModule;
import org.elasticsearch.index.similarity.*;
import org.testng.annotations.Test;
import java.io.IOException;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.MatcherAssert.assertThat;
public class SimilarityTests {
@Test
public void testResolveDefaultSimilarities() {
SimilarityLookupService similarityLookupService = similarityService().similarityLookupService();
assertThat(similarityLookupService.similarity("default"), instanceOf(PreBuiltSimilarityProvider.class));
assertThat(similarityLookupService.similarity("default").get(), instanceOf(DefaultSimilarity.class));
assertThat(similarityLookupService.similarity("BM25"), instanceOf(PreBuiltSimilarityProvider.class));
assertThat(similarityLookupService.similarity("BM25").get(), instanceOf(BM25Similarity.class));
}
@Test
public void testResolveSimilaritiesFromMapping_default() throws IOException {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("field1").field("type", "string").field("similarity", "my_similarity").endObject()
.endObject()
.endObject().endObject().string();
Settings indexSettings = ImmutableSettings.settingsBuilder()
.put("index.similarity.my_similarity.type", "default")
.put("index.similarity.my_similarity.discount_overlaps", false)
.build();
SimilarityService similarityService = similarityService(indexSettings);
DocumentMapper documentMapper = similarityService.mapperService().documentMapperParser().parse(mapping);
assertThat(documentMapper.mappers().name("field1").mapper().similarity(), instanceOf(DefaultSimilarityProvider.class));
DefaultSimilarity similarity = (DefaultSimilarity) documentMapper.mappers().name("field1").mapper().similarity().get();
assertThat(similarity.getDiscountOverlaps(), equalTo(false));
}
@Test
public void testResolveSimilaritiesFromMapping_bm25() throws IOException {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("field1").field("type", "string").field("similarity", "my_similarity").endObject()
.endObject()
.endObject().endObject().string();
Settings indexSettings = ImmutableSettings.settingsBuilder()
.put("index.similarity.my_similarity.type", "BM25")
.put("index.similarity.my_similarity.k1", 2.0f)
.put("index.similarity.my_similarity.b", 1.5f)
.put("index.similarity.my_similarity.discount_overlaps", false)
.build();
SimilarityService similarityService = similarityService(indexSettings);
DocumentMapper documentMapper = similarityService.mapperService().documentMapperParser().parse(mapping);
assertThat(documentMapper.mappers().name("field1").mapper().similarity(), instanceOf(BM25SimilarityProvider.class));
BM25Similarity similarity = (BM25Similarity) documentMapper.mappers().name("field1").mapper().similarity().get();
assertThat(similarity.getK1(), equalTo(2.0f));
assertThat(similarity.getB(), equalTo(1.5f));
assertThat(similarity.getDiscountOverlaps(), equalTo(false));
}
@Test
public void testResolveSimilaritiesFromMapping_DFR() throws IOException {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("field1").field("type", "string").field("similarity", "my_similarity").endObject()
.endObject()
.endObject().endObject().string();
Settings indexSettings = ImmutableSettings.settingsBuilder()
.put("index.similarity.my_similarity.type", "DFR")
.put("index.similarity.my_similarity.basic_model", "g")
.put("index.similarity.my_similarity.after_effect", "l")
.put("index.similarity.my_similarity.normalization", "h2")
.put("index.similarity.my_similarity.normalization.h2.c", 3f)
.build();
SimilarityService similarityService = similarityService(indexSettings);
DocumentMapper documentMapper = similarityService.mapperService().documentMapperParser().parse(mapping);
assertThat(documentMapper.mappers().name("field1").mapper().similarity(), instanceOf(DFRSimilarityProvider.class));
DFRSimilarity similarity = (DFRSimilarity) documentMapper.mappers().name("field1").mapper().similarity().get();
assertThat(similarity.getBasicModel(), instanceOf(BasicModelG.class));
assertThat(similarity.getAfterEffect(), instanceOf(AfterEffectL.class));
assertThat(similarity.getNormalization(), instanceOf(NormalizationH2.class));
assertThat(((NormalizationH2) similarity.getNormalization()).getC(), equalTo(3f));
}
@Test
public void testResolveSimilaritiesFromMapping_IB() throws IOException {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("field1").field("type", "string").field("similarity", "my_similarity").endObject()
.endObject()
.endObject().endObject().string();
Settings indexSettings = ImmutableSettings.settingsBuilder()
.put("index.similarity.my_similarity.type", "IB")
.put("index.similarity.my_similarity.distribution", "spl")
.put("index.similarity.my_similarity.lambda", "ttf")
.put("index.similarity.my_similarity.normalization", "h2")
.put("index.similarity.my_similarity.normalization.h2.c", 3f)
.build();
SimilarityService similarityService = similarityService(indexSettings);
DocumentMapper documentMapper = similarityService.mapperService().documentMapperParser().parse(mapping);
assertThat(documentMapper.mappers().name("field1").mapper().similarity(), instanceOf(IBSimilarityProvider.class));
IBSimilarity similarity = (IBSimilarity) documentMapper.mappers().name("field1").mapper().similarity().get();
assertThat(similarity.getDistribution(), instanceOf(DistributionSPL.class));
assertThat(similarity.getLambda(), instanceOf(LambdaTTF.class));
assertThat(similarity.getNormalization(), instanceOf(NormalizationH2.class));
assertThat(((NormalizationH2) similarity.getNormalization()).getC(), equalTo(3f));
}
private static SimilarityService similarityService() {
return similarityService(ImmutableSettings.Builder.EMPTY_SETTINGS);
}
private static SimilarityService similarityService(Settings settings) {
Index index = new Index("test");
Injector injector = new ModulesBuilder()
.add(new SettingsModule(settings))
.add(new IndexNameModule(index))
.add(new IndexSettingsModule(index, settings))
.add(new CodecModule(settings))
.add(new MapperServiceModule())
.add(new AnalysisModule(settings))
.add(new SimilarityModule(settings))
.createInjector();
return injector.getInstance(SimilarityService.class);
}
}