honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

remove fuzzy factor from mapping (internally implemented) 

we want to support ~ notion in query parser for types other than strings, we are getting there, one can do now age:10~5, we would love to support it for dates, as in timestamp:2012-10-10~5d, but that requires changes in the query parser to support strings after the ~ sign
This commit is contained in:
Shay Banon 2013-01-31 12:22:52 +01:00
parent 8df7f2af0d
commit 6cec73c201
18 changed files with 121 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java
View File

@ -128,6 +128,48 @@ public class MapperQueryParser extends QueryParser {
this.analyzeWildcard = settings.analyzeWildcard();
}
/**
* We override this one so we can get the fuzzy part to be treated as string, so people can do: "age:10~5". Note,
* we would love to support also "timestamp:2012-10-10~5d", but sadly the parser expects only numbers after the ~,
* hopefully we can change that in Lucene.
*/
@Override
Query handleBareTokenQuery(String qfield, Token term, Token fuzzySlop, boolean prefix, boolean wildcard, boolean fuzzy, boolean regexp) throws ParseException {
Query q;
String termImage = discardEscapeChar(term.image);
if (wildcard) {
q = getWildcardQuery(qfield, term.image);
} else if (prefix) {
q = getPrefixQuery(qfield,
discardEscapeChar(term.image.substring
(0, term.image.length() - 1)));
} else if (regexp) {
q = getRegexpQuery(qfield, term.image.substring(1, term.image.length() - 1));
} else if (fuzzy) {
// float fms = fuzzyMinSim;
// try {
// fms = Float.valueOf(fuzzySlop.image.substring(1)).floatValue();
// } catch (Exception ignored) {
// }
// if (fms < 0.0f) {
// throw new ParseException("Minimum similarity for a FuzzyQuery has to be between 0.0f and 1.0f !");
// } else if (fms >= 1.0f && fms != (int) fms) {
// throw new ParseException("Fractional edit distances are not allowed!");
// }
// q = getFuzzyQuery(qfield, termImage, fms);
if (fuzzySlop.image.length() == 1) {
q = getFuzzyQuery(qfield, termImage, Float.toString(fuzzyMinSim));
} else {
q = getFuzzyQuery(qfield, termImage, fuzzySlop.image.substring(1));
}
} else {
q = getFieldQuery(qfield, termImage, false);
}
return q;
}
@Override
protected Query newTermQuery(Term term) {
if (currentMapper != null) {
@ -295,9 +337,9 @@ public class MapperQueryParser extends QueryParser {
part2 = null;
}
if (lowercaseExpandedTerms) {
part1 = part1==null ? null : part1.toLowerCase(locale);
part2 = part2==null ? null : part2.toLowerCase(locale);
}
part1 = part1 == null ? null : part1.toLowerCase(locale);
part2 = part2 == null ? null : part2.toLowerCase(locale);
}
Collection<String> fields = extractMultiFields(field);
if (fields != null) {
if (fields.size() == 1) {
@ -356,8 +398,7 @@ public class MapperQueryParser extends QueryParser {
return newRangeQuery(field, part1, part2, startInclusive, endInclusive);
}
@Override
protected Query getFuzzyQuery(String field, String termStr, float minSimilarity) throws ParseException {
protected Query getFuzzyQuery(String field, String termStr, String minSimilarity) throws ParseException {
if (lowercaseExpandedTerms) {
termStr = termStr.toLowerCase(locale);
}
@ -395,7 +436,7 @@ public class MapperQueryParser extends QueryParser {
}
}
private Query getFuzzyQuerySingle(String field, String termStr, float minSimilarity) throws ParseException {
private Query getFuzzyQuerySingle(String field, String termStr, String minSimilarity) throws ParseException {
currentMapper = null;
MapperService.SmartNameFieldMappers fieldMappers = parseContext.smartFieldMappers(field);
if (fieldMappers != null) {
@ -413,7 +454,7 @@ public class MapperQueryParser extends QueryParser {
}
}
}
return super.getFuzzyQuery(field, termStr, minSimilarity);
return super.getFuzzyQuery(field, termStr, Float.parseFloat(minSimilarity));
}
@Override

2
src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
View File

@ -177,8 +177,6 @@ public interface FieldMapper<T> {
Query fuzzyQuery(String value, String minSim, int prefixLength, int maxExpansions, boolean transpositions);
Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions);
Query prefixQuery(Object value, @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryParseContext context);
Filter prefixFilter(Object value, @Nullable QueryParseContext context);

6
src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java
View File

@ -476,12 +476,6 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T>, Mapper {
return new FuzzyQuery(names.createIndexNameTerm(indexedValueForSearch(value)), edits, prefixLength, maxExpansions, transpositions);
}
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
int edits = FuzzyQuery.floatToEdits((float) minSim, value.codePointCount(0, value.length()));
return new FuzzyQuery(names.createIndexNameTerm(indexedValueForSearch(value)), edits, prefixLength, maxExpansions, transpositions);
}
@Override
public Query prefixQuery(Object value, @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryParseContext context) {
PrefixQuery query = new PrefixQuery(names().createIndexNameTerm(indexedValueForSearch(value)));

19
src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
View File

@ -89,7 +89,7 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
public ByteFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
ByteFieldMapper fieldMapper = new ByteFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, boost, fieldType, nullValue, ignoreMalformed(context),
precisionStep, boost, fieldType, nullValue, ignoreMalformed(context),
provider, similarity, fieldDataSettings);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
@ -116,9 +116,9 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
private String nullValueAsString;
protected ByteFieldMapper(Names names, int precisionStep, String fuzzyFactor, float boost, FieldType fieldType,
protected ByteFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
Byte nullValue, Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
super(names, precisionStep, fuzzyFactor, boost, fieldType,
super(names, precisionStep, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_byte/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)),
new NamedAnalyzer("_byte/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings);
this.nullValue = nullValue;
@ -190,16 +190,6 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
true, true);
}
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
byte iValue = Byte.parseByte(value);
byte iSim = (byte) (minSim * dFuzzyFactor);
return NumericRangeQuery.newIntRange(names.indexName(), precisionStep,
iValue - iSim,
iValue + iSim,
true, true);
}
@Override
public Query termQuery(Object value, @Nullable QueryParseContext context) {
int iValue = parseValue(value);
@ -351,9 +341,6 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
if (precisionStep != Defaults.PRECISION_STEP) {
builder.field("precision_step", precisionStep);
}
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (nullValue != null) {
builder.field("null_value", nullValue);
}

32
src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
View File

@ -116,7 +116,7 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
}
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter,
precisionStep, fuzzyFactor, boost, fieldType, nullValue,
precisionStep, boost, fieldType, nullValue,
timeUnit, parseUpperInclusive, ignoreMalformed(context), provider, similarity, fieldDataSettings);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
@ -153,11 +153,10 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
protected final TimeUnit timeUnit;
protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter, int precisionStep, String fuzzyFactor,
float boost, FieldType fieldType,
protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter, int precisionStep, float boost, FieldType fieldType,
String nullValue, TimeUnit timeUnit, boolean parseUpperInclusive, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
super(names, precisionStep, fuzzyFactor, boost, fieldType,
super(names, precisionStep, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_date/" + precisionStep,
new NumericDateAnalyzer(precisionStep, dateTimeFormatter.parser())),
new NamedAnalyzer("_date/max", new NumericDateAnalyzer(Integer.MAX_VALUE, dateTimeFormatter.parser())),
@ -179,18 +178,6 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
return new FieldDataType("long");
}
@Override
protected double parseFuzzyFactor(String fuzzyFactor) {
if (fuzzyFactor == null) {
return 1.0d;
}
try {
return TimeValue.parseTimeValue(fuzzyFactor, null).millis();
} catch (Exception e) {
return Double.parseDouble(fuzzyFactor);
}
}
@Override
protected int maxPrecisionStep() {
return 64;
@ -266,16 +253,6 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
true, true);
}
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
long iValue = dateMathParser.parse(value, System.currentTimeMillis());
long iSim = (long) (minSim * dFuzzyFactor);
return NumericRangeQuery.newLongRange(names.indexName(), precisionStep,
iValue - iSim,
iValue + iSim,
true, true);
}
@Override
public Query termQuery(Object value, @Nullable QueryParseContext context) {
long now = context == null ? System.currentTimeMillis() : context.nowInMillis();
@ -426,9 +403,6 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
if (precisionStep != Defaults.PRECISION_STEP) {
builder.field("precision_step", precisionStep);
}
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
builder.field("format", dateTimeFormatter.format());
if (nullValue != null) {
builder.field("null_value", nullValue);

20
src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
View File

@ -89,7 +89,7 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
public DoubleFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
DoubleFieldMapper fieldMapper = new DoubleFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, boost, fieldType, nullValue,
precisionStep, boost, fieldType, nullValue,
ignoreMalformed(context), provider, similarity, fieldDataSettings);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
@ -117,11 +117,10 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
private String nullValueAsString;
protected DoubleFieldMapper(Names names, int precisionStep, String fuzzyFactor,
float boost, FieldType fieldType,
protected DoubleFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
Double nullValue, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
super(names, precisionStep, fuzzyFactor, boost, fieldType,
super(names, precisionStep, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_double/" + precisionStep, new NumericDoubleAnalyzer(precisionStep)),
new NamedAnalyzer("_double/max", new NumericDoubleAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings);
this.nullValue = nullValue;
@ -185,16 +184,6 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
true, true);
}
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
double iValue = Double.parseDouble(value);
double iSim = minSim * dFuzzyFactor;
return NumericRangeQuery.newDoubleRange(names.indexName(), precisionStep,
iValue - iSim,
iValue + iSim,
true, true);
}
@Override
public Query termQuery(Object value, @Nullable QueryParseContext context) {
double dValue = parseValue(value);
@ -351,9 +340,6 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
if (precisionStep != Defaults.PRECISION_STEP) {
builder.field("precision_step", precisionStep);
}
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (nullValue != null) {
builder.field("null_value", nullValue);
}

19
src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
View File

@ -90,7 +90,7 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
public FloatFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
FloatFieldMapper fieldMapper = new FloatFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, boost, fieldType, nullValue,
precisionStep, boost, fieldType, nullValue,
ignoreMalformed(context), provider, similarity, fieldDataSettings);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
@ -117,9 +117,9 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
private String nullValueAsString;
protected FloatFieldMapper(Names names, int precisionStep, String fuzzyFactor, float boost, FieldType fieldType,
protected FloatFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
Float nullValue, Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
super(names, precisionStep, fuzzyFactor, boost, fieldType,
super(names, precisionStep, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_float/" + precisionStep, new NumericFloatAnalyzer(precisionStep)),
new NamedAnalyzer("_float/max", new NumericFloatAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings);
this.nullValue = nullValue;
@ -183,16 +183,6 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
true, true);
}
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
float iValue = Float.parseFloat(value);
float iSim = (float) (minSim * dFuzzyFactor);
return NumericRangeQuery.newFloatRange(names.indexName(), precisionStep,
iValue - iSim,
iValue + iSim,
true, true);
}
@Override
public Query termQuery(Object value, @Nullable QueryParseContext context) {
float fValue = parseValue(value);
@ -346,9 +336,6 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
if (precisionStep != Defaults.PRECISION_STEP) {
builder.field("precision_step", precisionStep);
}
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (nullValue != null) {
builder.field("null_value", nullValue);
}

21
src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
View File

@ -89,8 +89,7 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
@Override
public IntegerFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
IntegerFieldMapper fieldMapper = new IntegerFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, boost, fieldType,
IntegerFieldMapper fieldMapper = new IntegerFieldMapper(buildNames(context), precisionStep, boost, fieldType,
nullValue, ignoreMalformed(context), provider, similarity, fieldDataSettings);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
@ -117,11 +116,10 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
private String nullValueAsString;
protected IntegerFieldMapper(Names names, int precisionStep, String fuzzyFactor,
float boost, FieldType fieldType,
protected IntegerFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
Integer nullValue, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
super(names, precisionStep, fuzzyFactor, boost, fieldType,
super(names, precisionStep, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_int/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)),
new NamedAnalyzer("_int/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings);
this.nullValue = nullValue;
@ -189,16 +187,6 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
true, true);
}
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
int iValue = Integer.parseInt(value);
int iSim = (int) (minSim * dFuzzyFactor);
return NumericRangeQuery.newIntRange(names.indexName(), precisionStep,
iValue - iSim,
iValue + iSim,
true, true);
}
@Override
public Query termQuery(Object value, @Nullable QueryParseContext context) {
int iValue = parseValue(value);
@ -351,9 +339,6 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
if (precisionStep != Defaults.PRECISION_STEP) {
builder.field("precision_step", precisionStep);
}
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (nullValue != null) {
builder.field("null_value", nullValue);
}

20
src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
View File

@ -90,7 +90,7 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
public LongFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
LongFieldMapper fieldMapper = new LongFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, boost, fieldType, nullValue,
precisionStep, boost, fieldType, nullValue,
ignoreMalformed(context), provider, similarity, fieldDataSettings);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
@ -117,11 +117,10 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
private String nullValueAsString;
protected LongFieldMapper(Names names, int precisionStep, String fuzzyFactor,
float boost, FieldType fieldType,
protected LongFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
Long nullValue, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
super(names, precisionStep, fuzzyFactor, boost, fieldType,
super(names, precisionStep, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_long/" + precisionStep, new NumericLongAnalyzer(precisionStep)),
new NamedAnalyzer("_long/max", new NumericLongAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings);
this.nullValue = nullValue;
@ -189,16 +188,6 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
true, true);
}
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
long iValue = Long.parseLong(value);
long iSim = (long) (minSim * dFuzzyFactor);
return NumericRangeQuery.newLongRange(names.indexName(), precisionStep,
iValue - iSim,
iValue + iSim,
true, true);
}
@Override
public Query termQuery(Object value, @Nullable QueryParseContext context) {
long iValue = parseValue(value);
@ -350,9 +339,6 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
if (precisionStep != Defaults.PRECISION_STEP) {
builder.field("precision_step", precisionStep);
}
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (nullValue != null) {
builder.field("null_value", nullValue);
}

29
src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
View File

@ -59,7 +59,6 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
FIELD_TYPE.freeze();
}
public static final String FUZZY_FACTOR = null;
public static final Explicit<Boolean> IGNORE_MALFORMED = new Explicit<Boolean>(false, false);
}
@ -67,8 +66,6 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
protected int precisionStep = Defaults.PRECISION_STEP;
protected String fuzzyFactor = Defaults.FUZZY_FACTOR;
private Boolean ignoreMalformed;
public Builder(String name, FieldType fieldType) {
@ -100,11 +97,6 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
return builder;
}
public T fuzzyFactor(String fuzzyFactor) {
this.fuzzyFactor = fuzzyFactor;
return builder;
}
public T ignoreMalformed(boolean ignoreMalformed) {
this.ignoreMalformed = ignoreMalformed;
return builder;
@ -123,10 +115,6 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
protected int precisionStep;
protected String fuzzyFactor;
protected double dFuzzyFactor;
protected Boolean includeInAll;
protected Explicit<Boolean> ignoreMalformed;
@ -138,8 +126,7 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
}
};
protected NumberFieldMapper(Names names, int precisionStep, @Nullable String fuzzyFactor,
float boost, FieldType fieldType,
protected NumberFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
Explicit<Boolean> ignoreMalformed, NamedAnalyzer indexAnalyzer,
NamedAnalyzer searchAnalyzer, PostingsFormatProvider provider, SimilarityProvider similarity,
@Nullable Settings fieldDataSettings) {
@ -150,18 +137,9 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
} else {
this.precisionStep = precisionStep;
}
this.fuzzyFactor = fuzzyFactor;
this.dFuzzyFactor = parseFuzzyFactor(fuzzyFactor);
this.ignoreMalformed = ignoreMalformed;
}
protected double parseFuzzyFactor(String fuzzyFactor) {
if (fuzzyFactor == null) {
return 1.0d;
}
return Double.parseDouble(fuzzyFactor);
}
@Override
public void includeInAll(Boolean includeInAll) {
if (includeInAll != null) {
@ -237,9 +215,6 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
@Override
public abstract Query fuzzyQuery(String value, String minSim, int prefixLength, int maxExpansions, boolean transpositions);
@Override
public abstract Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions);
/**
* A range filter based on the field data cache.
*/
@ -265,8 +240,6 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
NumberFieldMapper nfmMergeWith = (NumberFieldMapper) mergeWith;
this.precisionStep = nfmMergeWith.precisionStep;
this.includeInAll = nfmMergeWith.includeInAll;
this.fuzzyFactor = nfmMergeWith.fuzzyFactor;
this.dFuzzyFactor = parseFuzzyFactor(nfmMergeWith.fuzzyFactor);
if (nfmMergeWith.ignoreMalformed.explicit()) {
this.ignoreMalformed = nfmMergeWith.ignoreMalformed;
}

20
src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
View File

@ -90,7 +90,7 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
public ShortFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
ShortFieldMapper fieldMapper = new ShortFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, boost, fieldType, nullValue,
precisionStep, boost, fieldType, nullValue,
ignoreMalformed(context), provider, similarity, fieldDataSettings);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
@ -117,11 +117,10 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
private String nullValueAsString;
protected ShortFieldMapper(Names names, int precisionStep, String fuzzyFactor,
float boost, FieldType fieldType,
protected ShortFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
Short nullValue, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
super(names, precisionStep, fuzzyFactor, boost, fieldType,
super(names, precisionStep, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_short/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)),
new NamedAnalyzer("_short/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings);
this.nullValue = nullValue;
@ -193,16 +192,6 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
true, true);
}
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
short iValue = Short.parseShort(value);
short iSim = (short) (minSim * dFuzzyFactor);
return NumericRangeQuery.newIntRange(names.indexName(), precisionStep,
iValue - iSim,
iValue + iSim,
true, true);
}
@Override
public Query termQuery(Object value, @Nullable QueryParseContext context) {
int iValue = parseValueAsInt(value);
@ -354,9 +343,6 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
if (precisionStep != Defaults.PRECISION_STEP) {
builder.field("precision_step", precisionStep);
}
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (nullValue != null) {
builder.field("null_value", nullValue);
}

2
src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java
View File

@ -50,8 +50,6 @@ public class TypeParsers {
Object propNode = entry.getValue();
if (propName.equals("precision_step")) {
builder.precisionStep(nodeIntegerValue(propNode));
} else if (propName.equals("fuzzy_factor")) {
builder.fuzzyFactor(propNode.toString());
} else if (propName.equals("ignore_malformed")) {
builder.ignoreMalformed(nodeBooleanValue(propNode));
} else if (propName.equals("omit_norms")) {

12
src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java
View File

@ -122,7 +122,7 @@ public class BoostFieldMapper extends NumberFieldMapper<Float> implements Intern
protected BoostFieldMapper(String name, String indexName, int precisionStep, float boost, FieldType fieldType,
Float nullValue, PostingsFormatProvider provider, @Nullable Settings fieldDataSettings) {
super(new Names(name, indexName, indexName, name), precisionStep, null, boost, fieldType,
super(new Names(name, indexName, indexName, name), precisionStep, boost, fieldType,
Defaults.IGNORE_MALFORMED, new NamedAnalyzer("_float/" + precisionStep, new NumericFloatAnalyzer(precisionStep)),
new NamedAnalyzer("_float/max", new NumericFloatAnalyzer(Integer.MAX_VALUE)), provider, null, fieldDataSettings);
this.nullValue = nullValue;
@ -185,16 +185,6 @@ public class BoostFieldMapper extends NumberFieldMapper<Float> implements Intern
true, true);
}
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
float iValue = Float.parseFloat(value);
float iSim = (float) (minSim * dFuzzyFactor);
return NumericRangeQuery.newFloatRange(names.indexName(), precisionStep,
iValue - iSim,
iValue + iSim,
true, true);
}
@Override
public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
return NumericRangeQuery.newFloatRange(names.indexName(), precisionStep,

2
src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java
View File

@ -101,7 +101,7 @@ public class SizeFieldMapper extends IntegerFieldMapper implements RootMapper {
}
public SizeFieldMapper(boolean enabled, FieldType fieldType, PostingsFormatProvider provider, @Nullable Settings fieldDataSettings) {
super(new Names(Defaults.NAME), Defaults.PRECISION_STEP, Defaults.FUZZY_FACTOR,
super(new Names(Defaults.NAME), Defaults.PRECISION_STEP,
Defaults.BOOST, fieldType, Defaults.NULL_VALUE, Defaults.IGNORE_MALFORMED, provider, null, fieldDataSettings);
this.enabled = enabled;
}

2
src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
View File

@ -120,7 +120,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R
protected TTLFieldMapper(FieldType fieldType, boolean enabled, long defaultTTL, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider, @Nullable Settings fieldDataSettings) {
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), Defaults.PRECISION_STEP,
Defaults.FUZZY_FACTOR, Defaults.BOOST, fieldType, Defaults.NULL_VALUE, ignoreMalformed,
Defaults.BOOST, fieldType, Defaults.NULL_VALUE, ignoreMalformed,
provider, null, fieldDataSettings);
this.enabled = enabled;
this.defaultTTL = defaultTTL;

2
src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
View File

@ -138,7 +138,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
FormatDateTimeFormatter dateTimeFormatter, boolean parseUpperInclusive,
Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider, @Nullable Settings fieldDataSettings) {
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter,
Defaults.PRECISION_STEP, Defaults.FUZZY_FACTOR, Defaults.BOOST, fieldType,
Defaults.PRECISION_STEP, Defaults.BOOST, fieldType,
Defaults.NULL_VALUE, TimeUnit.MILLISECONDS /*always milliseconds*/,
parseUpperInclusive, ignoreMalformed, provider, null, fieldDataSettings);
this.enabled = enabled;

17
src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
View File

@ -22,7 +22,10 @@ package org.elasticsearch.index.mapper.ip;
import org.apache.lucene.analysis.NumericTokenStream;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.search.*;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.NumericRangeFilter;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
@ -140,11 +143,10 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
private String nullValue;
protected IpFieldMapper(Names names, int precisionStep,
float boost, FieldType fieldType,
protected IpFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
String nullValue, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
super(names, precisionStep, null, boost, fieldType,
super(names, precisionStep, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)),
new NamedAnalyzer("_ip/max", new NumericIpAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings);
this.nullValue = nullValue;
@ -227,13 +229,6 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
true, true);
}
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
// Lucene 4 Upgrade: It's surprising this uses FuzzyQuery instead of NumericRangeQuery
int edits = FuzzyQuery.floatToEdits((float) minSim, value.codePointCount(0, value.length()));
return new FuzzyQuery(names.createIndexNameTerm(indexedValueForSearch(value)), edits, prefixLength, maxExpansions, transpositions);
}
@Override
public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
return NumericRangeQuery.newLongRange(names.indexName(), precisionStep,

51
src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java
View File

@ -105,7 +105,7 @@ public class SimpleQueryTests extends AbstractNodesTests {
assertTrue(e.getMessage().endsWith("IllegalStateException[field \"field1\" was indexed without position data; cannot run PhraseQuery (term=quick)]; }"));
}
}
@Test
public void testCommonTermsQuery() throws Exception {
try {
@ -126,40 +126,40 @@ public class SimpleQueryTests extends AbstractNodesTests {
assertThat(searchResponse.hits().totalHits(), equalTo(2l));
assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1"));
assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2"));
searchResponse = client.prepareSearch().setQuery(QueryBuilders.commonTerms("field1", "the quick brown").cutoffFrequency(3).lowFreqOperator(Operator.OR)).execute().actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(3l));
assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1"));
assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2"));
assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("3"));
searchResponse = client.prepareSearch().setQuery(QueryBuilders.commonTerms("field1", "the quick brown").cutoffFrequency(3).analyzer("standard")).execute().actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(3l));
// standard drops "the" since its a stopword
assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1"));
assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("3"));
assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("2"));
// try the same with match query
searchResponse = client.prepareSearch().setQuery(QueryBuilders.matchQuery("field1", "the quick brown").cutoffFrequency(3).operator(MatchQueryBuilder.Operator.AND)).execute().actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(2l));
assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1"));
assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2"));
searchResponse = client.prepareSearch().setQuery(QueryBuilders.matchQuery("field1", "the quick brown").cutoffFrequency(3).operator(MatchQueryBuilder.Operator.OR)).execute().actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(3l));
assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1"));
assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2"));
assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("3"));
searchResponse = client.prepareSearch().setQuery(QueryBuilders.matchQuery("field1", "the quick brown").cutoffFrequency(3).operator(MatchQueryBuilder.Operator.AND).analyzer("standard")).execute().actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(3l));
// standard drops "the" since its a stopword
assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1"));
assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("3"));
assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("2"));
// try the same with multi match query
searchResponse = client.prepareSearch().setQuery(QueryBuilders.multiMatchQuery("the quick brown", "field1", "field2").cutoffFrequency(3).operator(MatchQueryBuilder.Operator.AND)).execute().actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(3l));
@ -167,7 +167,7 @@ public class SimpleQueryTests extends AbstractNodesTests {
assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("1"));
assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("2"));
}
@Test
public void testOmitTermFreqsAndPositions() throws Exception {
// backwards compat test!
@ -223,7 +223,7 @@ public class SimpleQueryTests extends AbstractNodesTests {
searchResponse = client.prepareSearch().setQuery(queryString("v?l*e?1").analyzeWildcard(true)).execute().actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(1l));
}
@Test
public void testLowercaseExpandedTerms() {
try {
@ -619,4 +619,35 @@ public class SimpleQueryTests extends AbstractNodesTests {
assertThat(searchResponse.hits().totalHits(), equalTo(2l));
}
@Test
public void testFuzzyQueryString() {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("index.number_of_shards", 1)).execute().actionGet();
client.prepareIndex("test", "type1", "1").setSource("str", "kimchy", "date", "2012-02-01", "num", 12).execute().actionGet();
client.prepareIndex("test", "type1", "2").setSource("str", "shay", "date", "2012-02-05", "num", 20).execute().actionGet();
client.admin().indices().prepareRefresh().execute().actionGet();
SearchResponse searchResponse = client.prepareSearch()
.setQuery(queryString("str:kimcy~1"))
.execute().actionGet();
assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0));
assertThat(searchResponse.hits().totalHits(), equalTo(1l));
assertThat(searchResponse.hits().getAt(0).id(), equalTo("1"));
searchResponse = client.prepareSearch()
.setQuery(queryString("num:11~1"))
.execute().actionGet();
assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0));
assertThat(searchResponse.hits().totalHits(), equalTo(1l));
assertThat(searchResponse.hits().getAt(0).id(), equalTo("1"));
// Note, this test fails, i.e returns 0 results, the reason is that Lucene QP only supports numbers after the ~
// once this is changed in lucene to support strings, then this test will fail (good!)
searchResponse = client.prepareSearch()
.setQuery(queryString("date:2012-02-02~1d"))
.execute().actionGet();
assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0));
assertThat(searchResponse.hits().totalHits(), equalTo(0l));
}
}