remove fuzzy factor from mapping (internally implemented)

we want to support ~ notion in query parser for types other than strings, we are getting there, one can do now age:10~5, we would love to support it for dates, as in timestamp:2012-10-10~5d, but that requires changes in the query parser to support strings after the ~ sign
This commit is contained in:
Shay Banon 2013-01-31 12:22:52 +01:00
parent 8df7f2af0d
commit 6cec73c201
18 changed files with 121 additions and 210 deletions

View File

@ -128,6 +128,48 @@ public class MapperQueryParser extends QueryParser {
this.analyzeWildcard = settings.analyzeWildcard(); this.analyzeWildcard = settings.analyzeWildcard();
} }
/**
* We override this one so we can get the fuzzy part to be treated as string, so people can do: "age:10~5". Note,
* we would love to support also "timestamp:2012-10-10~5d", but sadly the parser expects only numbers after the ~,
* hopefully we can change that in Lucene.
*/
@Override
Query handleBareTokenQuery(String qfield, Token term, Token fuzzySlop, boolean prefix, boolean wildcard, boolean fuzzy, boolean regexp) throws ParseException {
Query q;
String termImage = discardEscapeChar(term.image);
if (wildcard) {
q = getWildcardQuery(qfield, term.image);
} else if (prefix) {
q = getPrefixQuery(qfield,
discardEscapeChar(term.image.substring
(0, term.image.length() - 1)));
} else if (regexp) {
q = getRegexpQuery(qfield, term.image.substring(1, term.image.length() - 1));
} else if (fuzzy) {
// float fms = fuzzyMinSim;
// try {
// fms = Float.valueOf(fuzzySlop.image.substring(1)).floatValue();
// } catch (Exception ignored) {
// }
// if (fms < 0.0f) {
// throw new ParseException("Minimum similarity for a FuzzyQuery has to be between 0.0f and 1.0f !");
// } else if (fms >= 1.0f && fms != (int) fms) {
// throw new ParseException("Fractional edit distances are not allowed!");
// }
// q = getFuzzyQuery(qfield, termImage, fms);
if (fuzzySlop.image.length() == 1) {
q = getFuzzyQuery(qfield, termImage, Float.toString(fuzzyMinSim));
} else {
q = getFuzzyQuery(qfield, termImage, fuzzySlop.image.substring(1));
}
} else {
q = getFieldQuery(qfield, termImage, false);
}
return q;
}
@Override @Override
protected Query newTermQuery(Term term) { protected Query newTermQuery(Term term) {
if (currentMapper != null) { if (currentMapper != null) {
@ -356,8 +398,7 @@ public class MapperQueryParser extends QueryParser {
return newRangeQuery(field, part1, part2, startInclusive, endInclusive); return newRangeQuery(field, part1, part2, startInclusive, endInclusive);
} }
@Override protected Query getFuzzyQuery(String field, String termStr, String minSimilarity) throws ParseException {
protected Query getFuzzyQuery(String field, String termStr, float minSimilarity) throws ParseException {
if (lowercaseExpandedTerms) { if (lowercaseExpandedTerms) {
termStr = termStr.toLowerCase(locale); termStr = termStr.toLowerCase(locale);
} }
@ -395,7 +436,7 @@ public class MapperQueryParser extends QueryParser {
} }
} }
private Query getFuzzyQuerySingle(String field, String termStr, float minSimilarity) throws ParseException { private Query getFuzzyQuerySingle(String field, String termStr, String minSimilarity) throws ParseException {
currentMapper = null; currentMapper = null;
MapperService.SmartNameFieldMappers fieldMappers = parseContext.smartFieldMappers(field); MapperService.SmartNameFieldMappers fieldMappers = parseContext.smartFieldMappers(field);
if (fieldMappers != null) { if (fieldMappers != null) {
@ -413,7 +454,7 @@ public class MapperQueryParser extends QueryParser {
} }
} }
} }
return super.getFuzzyQuery(field, termStr, minSimilarity); return super.getFuzzyQuery(field, termStr, Float.parseFloat(minSimilarity));
} }
@Override @Override

View File

@ -177,8 +177,6 @@ public interface FieldMapper<T> {
Query fuzzyQuery(String value, String minSim, int prefixLength, int maxExpansions, boolean transpositions); Query fuzzyQuery(String value, String minSim, int prefixLength, int maxExpansions, boolean transpositions);
Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions);
Query prefixQuery(Object value, @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryParseContext context); Query prefixQuery(Object value, @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryParseContext context);
Filter prefixFilter(Object value, @Nullable QueryParseContext context); Filter prefixFilter(Object value, @Nullable QueryParseContext context);

View File

@ -476,12 +476,6 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T>, Mapper {
return new FuzzyQuery(names.createIndexNameTerm(indexedValueForSearch(value)), edits, prefixLength, maxExpansions, transpositions); return new FuzzyQuery(names.createIndexNameTerm(indexedValueForSearch(value)), edits, prefixLength, maxExpansions, transpositions);
} }
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
int edits = FuzzyQuery.floatToEdits((float) minSim, value.codePointCount(0, value.length()));
return new FuzzyQuery(names.createIndexNameTerm(indexedValueForSearch(value)), edits, prefixLength, maxExpansions, transpositions);
}
@Override @Override
public Query prefixQuery(Object value, @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryParseContext context) { public Query prefixQuery(Object value, @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryParseContext context) {
PrefixQuery query = new PrefixQuery(names().createIndexNameTerm(indexedValueForSearch(value))); PrefixQuery query = new PrefixQuery(names().createIndexNameTerm(indexedValueForSearch(value)));

View File

@ -89,7 +89,7 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
public ByteFieldMapper build(BuilderContext context) { public ByteFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
ByteFieldMapper fieldMapper = new ByteFieldMapper(buildNames(context), ByteFieldMapper fieldMapper = new ByteFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, boost, fieldType, nullValue, ignoreMalformed(context), precisionStep, boost, fieldType, nullValue, ignoreMalformed(context),
provider, similarity, fieldDataSettings); provider, similarity, fieldDataSettings);
fieldMapper.includeInAll(includeInAll); fieldMapper.includeInAll(includeInAll);
return fieldMapper; return fieldMapper;
@ -116,9 +116,9 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
private String nullValueAsString; private String nullValueAsString;
protected ByteFieldMapper(Names names, int precisionStep, String fuzzyFactor, float boost, FieldType fieldType, protected ByteFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
Byte nullValue, Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) { Byte nullValue, Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
super(names, precisionStep, fuzzyFactor, boost, fieldType, super(names, precisionStep, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_byte/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)), ignoreMalformed, new NamedAnalyzer("_byte/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)),
new NamedAnalyzer("_byte/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings); new NamedAnalyzer("_byte/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings);
this.nullValue = nullValue; this.nullValue = nullValue;
@ -190,16 +190,6 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
true, true); true, true);
} }
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
byte iValue = Byte.parseByte(value);
byte iSim = (byte) (minSim * dFuzzyFactor);
return NumericRangeQuery.newIntRange(names.indexName(), precisionStep,
iValue - iSim,
iValue + iSim,
true, true);
}
@Override @Override
public Query termQuery(Object value, @Nullable QueryParseContext context) { public Query termQuery(Object value, @Nullable QueryParseContext context) {
int iValue = parseValue(value); int iValue = parseValue(value);
@ -351,9 +341,6 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
if (precisionStep != Defaults.PRECISION_STEP) { if (precisionStep != Defaults.PRECISION_STEP) {
builder.field("precision_step", precisionStep); builder.field("precision_step", precisionStep);
} }
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (nullValue != null) { if (nullValue != null) {
builder.field("null_value", nullValue); builder.field("null_value", nullValue);
} }

View File

@ -116,7 +116,7 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
} }
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter, DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter,
precisionStep, fuzzyFactor, boost, fieldType, nullValue, precisionStep, boost, fieldType, nullValue,
timeUnit, parseUpperInclusive, ignoreMalformed(context), provider, similarity, fieldDataSettings); timeUnit, parseUpperInclusive, ignoreMalformed(context), provider, similarity, fieldDataSettings);
fieldMapper.includeInAll(includeInAll); fieldMapper.includeInAll(includeInAll);
return fieldMapper; return fieldMapper;
@ -153,11 +153,10 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
protected final TimeUnit timeUnit; protected final TimeUnit timeUnit;
protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter, int precisionStep, String fuzzyFactor, protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter, int precisionStep, float boost, FieldType fieldType,
float boost, FieldType fieldType,
String nullValue, TimeUnit timeUnit, boolean parseUpperInclusive, Explicit<Boolean> ignoreMalformed, String nullValue, TimeUnit timeUnit, boolean parseUpperInclusive, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) { PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
super(names, precisionStep, fuzzyFactor, boost, fieldType, super(names, precisionStep, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_date/" + precisionStep, ignoreMalformed, new NamedAnalyzer("_date/" + precisionStep,
new NumericDateAnalyzer(precisionStep, dateTimeFormatter.parser())), new NumericDateAnalyzer(precisionStep, dateTimeFormatter.parser())),
new NamedAnalyzer("_date/max", new NumericDateAnalyzer(Integer.MAX_VALUE, dateTimeFormatter.parser())), new NamedAnalyzer("_date/max", new NumericDateAnalyzer(Integer.MAX_VALUE, dateTimeFormatter.parser())),
@ -179,18 +178,6 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
return new FieldDataType("long"); return new FieldDataType("long");
} }
@Override
protected double parseFuzzyFactor(String fuzzyFactor) {
if (fuzzyFactor == null) {
return 1.0d;
}
try {
return TimeValue.parseTimeValue(fuzzyFactor, null).millis();
} catch (Exception e) {
return Double.parseDouble(fuzzyFactor);
}
}
@Override @Override
protected int maxPrecisionStep() { protected int maxPrecisionStep() {
return 64; return 64;
@ -266,16 +253,6 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
true, true); true, true);
} }
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
long iValue = dateMathParser.parse(value, System.currentTimeMillis());
long iSim = (long) (minSim * dFuzzyFactor);
return NumericRangeQuery.newLongRange(names.indexName(), precisionStep,
iValue - iSim,
iValue + iSim,
true, true);
}
@Override @Override
public Query termQuery(Object value, @Nullable QueryParseContext context) { public Query termQuery(Object value, @Nullable QueryParseContext context) {
long now = context == null ? System.currentTimeMillis() : context.nowInMillis(); long now = context == null ? System.currentTimeMillis() : context.nowInMillis();
@ -426,9 +403,6 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
if (precisionStep != Defaults.PRECISION_STEP) { if (precisionStep != Defaults.PRECISION_STEP) {
builder.field("precision_step", precisionStep); builder.field("precision_step", precisionStep);
} }
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
builder.field("format", dateTimeFormatter.format()); builder.field("format", dateTimeFormatter.format());
if (nullValue != null) { if (nullValue != null) {
builder.field("null_value", nullValue); builder.field("null_value", nullValue);

View File

@ -89,7 +89,7 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
public DoubleFieldMapper build(BuilderContext context) { public DoubleFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
DoubleFieldMapper fieldMapper = new DoubleFieldMapper(buildNames(context), DoubleFieldMapper fieldMapper = new DoubleFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, boost, fieldType, nullValue, precisionStep, boost, fieldType, nullValue,
ignoreMalformed(context), provider, similarity, fieldDataSettings); ignoreMalformed(context), provider, similarity, fieldDataSettings);
fieldMapper.includeInAll(includeInAll); fieldMapper.includeInAll(includeInAll);
return fieldMapper; return fieldMapper;
@ -117,11 +117,10 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
private String nullValueAsString; private String nullValueAsString;
protected DoubleFieldMapper(Names names, int precisionStep, String fuzzyFactor, protected DoubleFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
float boost, FieldType fieldType,
Double nullValue, Explicit<Boolean> ignoreMalformed, Double nullValue, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) { PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
super(names, precisionStep, fuzzyFactor, boost, fieldType, super(names, precisionStep, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_double/" + precisionStep, new NumericDoubleAnalyzer(precisionStep)), ignoreMalformed, new NamedAnalyzer("_double/" + precisionStep, new NumericDoubleAnalyzer(precisionStep)),
new NamedAnalyzer("_double/max", new NumericDoubleAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings); new NamedAnalyzer("_double/max", new NumericDoubleAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings);
this.nullValue = nullValue; this.nullValue = nullValue;
@ -185,16 +184,6 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
true, true); true, true);
} }
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
double iValue = Double.parseDouble(value);
double iSim = minSim * dFuzzyFactor;
return NumericRangeQuery.newDoubleRange(names.indexName(), precisionStep,
iValue - iSim,
iValue + iSim,
true, true);
}
@Override @Override
public Query termQuery(Object value, @Nullable QueryParseContext context) { public Query termQuery(Object value, @Nullable QueryParseContext context) {
double dValue = parseValue(value); double dValue = parseValue(value);
@ -351,9 +340,6 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
if (precisionStep != Defaults.PRECISION_STEP) { if (precisionStep != Defaults.PRECISION_STEP) {
builder.field("precision_step", precisionStep); builder.field("precision_step", precisionStep);
} }
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (nullValue != null) { if (nullValue != null) {
builder.field("null_value", nullValue); builder.field("null_value", nullValue);
} }

View File

@ -90,7 +90,7 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
public FloatFieldMapper build(BuilderContext context) { public FloatFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
FloatFieldMapper fieldMapper = new FloatFieldMapper(buildNames(context), FloatFieldMapper fieldMapper = new FloatFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, boost, fieldType, nullValue, precisionStep, boost, fieldType, nullValue,
ignoreMalformed(context), provider, similarity, fieldDataSettings); ignoreMalformed(context), provider, similarity, fieldDataSettings);
fieldMapper.includeInAll(includeInAll); fieldMapper.includeInAll(includeInAll);
return fieldMapper; return fieldMapper;
@ -117,9 +117,9 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
private String nullValueAsString; private String nullValueAsString;
protected FloatFieldMapper(Names names, int precisionStep, String fuzzyFactor, float boost, FieldType fieldType, protected FloatFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
Float nullValue, Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) { Float nullValue, Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
super(names, precisionStep, fuzzyFactor, boost, fieldType, super(names, precisionStep, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_float/" + precisionStep, new NumericFloatAnalyzer(precisionStep)), ignoreMalformed, new NamedAnalyzer("_float/" + precisionStep, new NumericFloatAnalyzer(precisionStep)),
new NamedAnalyzer("_float/max", new NumericFloatAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings); new NamedAnalyzer("_float/max", new NumericFloatAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings);
this.nullValue = nullValue; this.nullValue = nullValue;
@ -183,16 +183,6 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
true, true); true, true);
} }
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
float iValue = Float.parseFloat(value);
float iSim = (float) (minSim * dFuzzyFactor);
return NumericRangeQuery.newFloatRange(names.indexName(), precisionStep,
iValue - iSim,
iValue + iSim,
true, true);
}
@Override @Override
public Query termQuery(Object value, @Nullable QueryParseContext context) { public Query termQuery(Object value, @Nullable QueryParseContext context) {
float fValue = parseValue(value); float fValue = parseValue(value);
@ -346,9 +336,6 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
if (precisionStep != Defaults.PRECISION_STEP) { if (precisionStep != Defaults.PRECISION_STEP) {
builder.field("precision_step", precisionStep); builder.field("precision_step", precisionStep);
} }
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (nullValue != null) { if (nullValue != null) {
builder.field("null_value", nullValue); builder.field("null_value", nullValue);
} }

View File

@ -89,8 +89,7 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
@Override @Override
public IntegerFieldMapper build(BuilderContext context) { public IntegerFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
IntegerFieldMapper fieldMapper = new IntegerFieldMapper(buildNames(context), IntegerFieldMapper fieldMapper = new IntegerFieldMapper(buildNames(context), precisionStep, boost, fieldType,
precisionStep, fuzzyFactor, boost, fieldType,
nullValue, ignoreMalformed(context), provider, similarity, fieldDataSettings); nullValue, ignoreMalformed(context), provider, similarity, fieldDataSettings);
fieldMapper.includeInAll(includeInAll); fieldMapper.includeInAll(includeInAll);
return fieldMapper; return fieldMapper;
@ -117,11 +116,10 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
private String nullValueAsString; private String nullValueAsString;
protected IntegerFieldMapper(Names names, int precisionStep, String fuzzyFactor, protected IntegerFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
float boost, FieldType fieldType,
Integer nullValue, Explicit<Boolean> ignoreMalformed, Integer nullValue, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) { PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
super(names, precisionStep, fuzzyFactor, boost, fieldType, super(names, precisionStep, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_int/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)), ignoreMalformed, new NamedAnalyzer("_int/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)),
new NamedAnalyzer("_int/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings); new NamedAnalyzer("_int/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings);
this.nullValue = nullValue; this.nullValue = nullValue;
@ -189,16 +187,6 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
true, true); true, true);
} }
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
int iValue = Integer.parseInt(value);
int iSim = (int) (minSim * dFuzzyFactor);
return NumericRangeQuery.newIntRange(names.indexName(), precisionStep,
iValue - iSim,
iValue + iSim,
true, true);
}
@Override @Override
public Query termQuery(Object value, @Nullable QueryParseContext context) { public Query termQuery(Object value, @Nullable QueryParseContext context) {
int iValue = parseValue(value); int iValue = parseValue(value);
@ -351,9 +339,6 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
if (precisionStep != Defaults.PRECISION_STEP) { if (precisionStep != Defaults.PRECISION_STEP) {
builder.field("precision_step", precisionStep); builder.field("precision_step", precisionStep);
} }
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (nullValue != null) { if (nullValue != null) {
builder.field("null_value", nullValue); builder.field("null_value", nullValue);
} }

View File

@ -90,7 +90,7 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
public LongFieldMapper build(BuilderContext context) { public LongFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
LongFieldMapper fieldMapper = new LongFieldMapper(buildNames(context), LongFieldMapper fieldMapper = new LongFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, boost, fieldType, nullValue, precisionStep, boost, fieldType, nullValue,
ignoreMalformed(context), provider, similarity, fieldDataSettings); ignoreMalformed(context), provider, similarity, fieldDataSettings);
fieldMapper.includeInAll(includeInAll); fieldMapper.includeInAll(includeInAll);
return fieldMapper; return fieldMapper;
@ -117,11 +117,10 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
private String nullValueAsString; private String nullValueAsString;
protected LongFieldMapper(Names names, int precisionStep, String fuzzyFactor, protected LongFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
float boost, FieldType fieldType,
Long nullValue, Explicit<Boolean> ignoreMalformed, Long nullValue, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) { PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
super(names, precisionStep, fuzzyFactor, boost, fieldType, super(names, precisionStep, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_long/" + precisionStep, new NumericLongAnalyzer(precisionStep)), ignoreMalformed, new NamedAnalyzer("_long/" + precisionStep, new NumericLongAnalyzer(precisionStep)),
new NamedAnalyzer("_long/max", new NumericLongAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings); new NamedAnalyzer("_long/max", new NumericLongAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings);
this.nullValue = nullValue; this.nullValue = nullValue;
@ -189,16 +188,6 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
true, true); true, true);
} }
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
long iValue = Long.parseLong(value);
long iSim = (long) (minSim * dFuzzyFactor);
return NumericRangeQuery.newLongRange(names.indexName(), precisionStep,
iValue - iSim,
iValue + iSim,
true, true);
}
@Override @Override
public Query termQuery(Object value, @Nullable QueryParseContext context) { public Query termQuery(Object value, @Nullable QueryParseContext context) {
long iValue = parseValue(value); long iValue = parseValue(value);
@ -350,9 +339,6 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
if (precisionStep != Defaults.PRECISION_STEP) { if (precisionStep != Defaults.PRECISION_STEP) {
builder.field("precision_step", precisionStep); builder.field("precision_step", precisionStep);
} }
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (nullValue != null) { if (nullValue != null) {
builder.field("null_value", nullValue); builder.field("null_value", nullValue);
} }

View File

@ -59,7 +59,6 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
FIELD_TYPE.freeze(); FIELD_TYPE.freeze();
} }
public static final String FUZZY_FACTOR = null;
public static final Explicit<Boolean> IGNORE_MALFORMED = new Explicit<Boolean>(false, false); public static final Explicit<Boolean> IGNORE_MALFORMED = new Explicit<Boolean>(false, false);
} }
@ -67,8 +66,6 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
protected int precisionStep = Defaults.PRECISION_STEP; protected int precisionStep = Defaults.PRECISION_STEP;
protected String fuzzyFactor = Defaults.FUZZY_FACTOR;
private Boolean ignoreMalformed; private Boolean ignoreMalformed;
public Builder(String name, FieldType fieldType) { public Builder(String name, FieldType fieldType) {
@ -100,11 +97,6 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
return builder; return builder;
} }
public T fuzzyFactor(String fuzzyFactor) {
this.fuzzyFactor = fuzzyFactor;
return builder;
}
public T ignoreMalformed(boolean ignoreMalformed) { public T ignoreMalformed(boolean ignoreMalformed) {
this.ignoreMalformed = ignoreMalformed; this.ignoreMalformed = ignoreMalformed;
return builder; return builder;
@ -123,10 +115,6 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
protected int precisionStep; protected int precisionStep;
protected String fuzzyFactor;
protected double dFuzzyFactor;
protected Boolean includeInAll; protected Boolean includeInAll;
protected Explicit<Boolean> ignoreMalformed; protected Explicit<Boolean> ignoreMalformed;
@ -138,8 +126,7 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
} }
}; };
protected NumberFieldMapper(Names names, int precisionStep, @Nullable String fuzzyFactor, protected NumberFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
float boost, FieldType fieldType,
Explicit<Boolean> ignoreMalformed, NamedAnalyzer indexAnalyzer, Explicit<Boolean> ignoreMalformed, NamedAnalyzer indexAnalyzer,
NamedAnalyzer searchAnalyzer, PostingsFormatProvider provider, SimilarityProvider similarity, NamedAnalyzer searchAnalyzer, PostingsFormatProvider provider, SimilarityProvider similarity,
@Nullable Settings fieldDataSettings) { @Nullable Settings fieldDataSettings) {
@ -150,18 +137,9 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
} else { } else {
this.precisionStep = precisionStep; this.precisionStep = precisionStep;
} }
this.fuzzyFactor = fuzzyFactor;
this.dFuzzyFactor = parseFuzzyFactor(fuzzyFactor);
this.ignoreMalformed = ignoreMalformed; this.ignoreMalformed = ignoreMalformed;
} }
protected double parseFuzzyFactor(String fuzzyFactor) {
if (fuzzyFactor == null) {
return 1.0d;
}
return Double.parseDouble(fuzzyFactor);
}
@Override @Override
public void includeInAll(Boolean includeInAll) { public void includeInAll(Boolean includeInAll) {
if (includeInAll != null) { if (includeInAll != null) {
@ -237,9 +215,6 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
@Override @Override
public abstract Query fuzzyQuery(String value, String minSim, int prefixLength, int maxExpansions, boolean transpositions); public abstract Query fuzzyQuery(String value, String minSim, int prefixLength, int maxExpansions, boolean transpositions);
@Override
public abstract Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions);
/** /**
* A range filter based on the field data cache. * A range filter based on the field data cache.
*/ */
@ -265,8 +240,6 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
NumberFieldMapper nfmMergeWith = (NumberFieldMapper) mergeWith; NumberFieldMapper nfmMergeWith = (NumberFieldMapper) mergeWith;
this.precisionStep = nfmMergeWith.precisionStep; this.precisionStep = nfmMergeWith.precisionStep;
this.includeInAll = nfmMergeWith.includeInAll; this.includeInAll = nfmMergeWith.includeInAll;
this.fuzzyFactor = nfmMergeWith.fuzzyFactor;
this.dFuzzyFactor = parseFuzzyFactor(nfmMergeWith.fuzzyFactor);
if (nfmMergeWith.ignoreMalformed.explicit()) { if (nfmMergeWith.ignoreMalformed.explicit()) {
this.ignoreMalformed = nfmMergeWith.ignoreMalformed; this.ignoreMalformed = nfmMergeWith.ignoreMalformed;
} }

View File

@ -90,7 +90,7 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
public ShortFieldMapper build(BuilderContext context) { public ShortFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
ShortFieldMapper fieldMapper = new ShortFieldMapper(buildNames(context), ShortFieldMapper fieldMapper = new ShortFieldMapper(buildNames(context),
precisionStep, fuzzyFactor, boost, fieldType, nullValue, precisionStep, boost, fieldType, nullValue,
ignoreMalformed(context), provider, similarity, fieldDataSettings); ignoreMalformed(context), provider, similarity, fieldDataSettings);
fieldMapper.includeInAll(includeInAll); fieldMapper.includeInAll(includeInAll);
return fieldMapper; return fieldMapper;
@ -117,11 +117,10 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
private String nullValueAsString; private String nullValueAsString;
protected ShortFieldMapper(Names names, int precisionStep, String fuzzyFactor, protected ShortFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
float boost, FieldType fieldType,
Short nullValue, Explicit<Boolean> ignoreMalformed, Short nullValue, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) { PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
super(names, precisionStep, fuzzyFactor, boost, fieldType, super(names, precisionStep, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_short/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)), ignoreMalformed, new NamedAnalyzer("_short/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)),
new NamedAnalyzer("_short/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings); new NamedAnalyzer("_short/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings);
this.nullValue = nullValue; this.nullValue = nullValue;
@ -193,16 +192,6 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
true, true); true, true);
} }
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
short iValue = Short.parseShort(value);
short iSim = (short) (minSim * dFuzzyFactor);
return NumericRangeQuery.newIntRange(names.indexName(), precisionStep,
iValue - iSim,
iValue + iSim,
true, true);
}
@Override @Override
public Query termQuery(Object value, @Nullable QueryParseContext context) { public Query termQuery(Object value, @Nullable QueryParseContext context) {
int iValue = parseValueAsInt(value); int iValue = parseValueAsInt(value);
@ -354,9 +343,6 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
if (precisionStep != Defaults.PRECISION_STEP) { if (precisionStep != Defaults.PRECISION_STEP) {
builder.field("precision_step", precisionStep); builder.field("precision_step", precisionStep);
} }
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
builder.field("fuzzy_factor", fuzzyFactor);
}
if (nullValue != null) { if (nullValue != null) {
builder.field("null_value", nullValue); builder.field("null_value", nullValue);
} }

View File

@ -50,8 +50,6 @@ public class TypeParsers {
Object propNode = entry.getValue(); Object propNode = entry.getValue();
if (propName.equals("precision_step")) { if (propName.equals("precision_step")) {
builder.precisionStep(nodeIntegerValue(propNode)); builder.precisionStep(nodeIntegerValue(propNode));
} else if (propName.equals("fuzzy_factor")) {
builder.fuzzyFactor(propNode.toString());
} else if (propName.equals("ignore_malformed")) { } else if (propName.equals("ignore_malformed")) {
builder.ignoreMalformed(nodeBooleanValue(propNode)); builder.ignoreMalformed(nodeBooleanValue(propNode));
} else if (propName.equals("omit_norms")) { } else if (propName.equals("omit_norms")) {

View File

@ -122,7 +122,7 @@ public class BoostFieldMapper extends NumberFieldMapper<Float> implements Intern
protected BoostFieldMapper(String name, String indexName, int precisionStep, float boost, FieldType fieldType, protected BoostFieldMapper(String name, String indexName, int precisionStep, float boost, FieldType fieldType,
Float nullValue, PostingsFormatProvider provider, @Nullable Settings fieldDataSettings) { Float nullValue, PostingsFormatProvider provider, @Nullable Settings fieldDataSettings) {
super(new Names(name, indexName, indexName, name), precisionStep, null, boost, fieldType, super(new Names(name, indexName, indexName, name), precisionStep, boost, fieldType,
Defaults.IGNORE_MALFORMED, new NamedAnalyzer("_float/" + precisionStep, new NumericFloatAnalyzer(precisionStep)), Defaults.IGNORE_MALFORMED, new NamedAnalyzer("_float/" + precisionStep, new NumericFloatAnalyzer(precisionStep)),
new NamedAnalyzer("_float/max", new NumericFloatAnalyzer(Integer.MAX_VALUE)), provider, null, fieldDataSettings); new NamedAnalyzer("_float/max", new NumericFloatAnalyzer(Integer.MAX_VALUE)), provider, null, fieldDataSettings);
this.nullValue = nullValue; this.nullValue = nullValue;
@ -185,16 +185,6 @@ public class BoostFieldMapper extends NumberFieldMapper<Float> implements Intern
true, true); true, true);
} }
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
float iValue = Float.parseFloat(value);
float iSim = (float) (minSim * dFuzzyFactor);
return NumericRangeQuery.newFloatRange(names.indexName(), precisionStep,
iValue - iSim,
iValue + iSim,
true, true);
}
@Override @Override
public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) { public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
return NumericRangeQuery.newFloatRange(names.indexName(), precisionStep, return NumericRangeQuery.newFloatRange(names.indexName(), precisionStep,

View File

@ -101,7 +101,7 @@ public class SizeFieldMapper extends IntegerFieldMapper implements RootMapper {
} }
public SizeFieldMapper(boolean enabled, FieldType fieldType, PostingsFormatProvider provider, @Nullable Settings fieldDataSettings) { public SizeFieldMapper(boolean enabled, FieldType fieldType, PostingsFormatProvider provider, @Nullable Settings fieldDataSettings) {
super(new Names(Defaults.NAME), Defaults.PRECISION_STEP, Defaults.FUZZY_FACTOR, super(new Names(Defaults.NAME), Defaults.PRECISION_STEP,
Defaults.BOOST, fieldType, Defaults.NULL_VALUE, Defaults.IGNORE_MALFORMED, provider, null, fieldDataSettings); Defaults.BOOST, fieldType, Defaults.NULL_VALUE, Defaults.IGNORE_MALFORMED, provider, null, fieldDataSettings);
this.enabled = enabled; this.enabled = enabled;
} }

View File

@ -120,7 +120,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R
protected TTLFieldMapper(FieldType fieldType, boolean enabled, long defaultTTL, Explicit<Boolean> ignoreMalformed, protected TTLFieldMapper(FieldType fieldType, boolean enabled, long defaultTTL, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider, @Nullable Settings fieldDataSettings) { PostingsFormatProvider provider, @Nullable Settings fieldDataSettings) {
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), Defaults.PRECISION_STEP, super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), Defaults.PRECISION_STEP,
Defaults.FUZZY_FACTOR, Defaults.BOOST, fieldType, Defaults.NULL_VALUE, ignoreMalformed, Defaults.BOOST, fieldType, Defaults.NULL_VALUE, ignoreMalformed,
provider, null, fieldDataSettings); provider, null, fieldDataSettings);
this.enabled = enabled; this.enabled = enabled;
this.defaultTTL = defaultTTL; this.defaultTTL = defaultTTL;

View File

@ -138,7 +138,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
FormatDateTimeFormatter dateTimeFormatter, boolean parseUpperInclusive, FormatDateTimeFormatter dateTimeFormatter, boolean parseUpperInclusive,
Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider, @Nullable Settings fieldDataSettings) { Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider, @Nullable Settings fieldDataSettings) {
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter, super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter,
Defaults.PRECISION_STEP, Defaults.FUZZY_FACTOR, Defaults.BOOST, fieldType, Defaults.PRECISION_STEP, Defaults.BOOST, fieldType,
Defaults.NULL_VALUE, TimeUnit.MILLISECONDS /*always milliseconds*/, Defaults.NULL_VALUE, TimeUnit.MILLISECONDS /*always milliseconds*/,
parseUpperInclusive, ignoreMalformed, provider, null, fieldDataSettings); parseUpperInclusive, ignoreMalformed, provider, null, fieldDataSettings);
this.enabled = enabled; this.enabled = enabled;

View File

@ -22,7 +22,10 @@ package org.elasticsearch.index.mapper.ip;
import org.apache.lucene.analysis.NumericTokenStream; import org.apache.lucene.analysis.NumericTokenStream;
import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType; import org.apache.lucene.document.FieldType;
import org.apache.lucene.search.*; import org.apache.lucene.search.Filter;
import org.apache.lucene.search.NumericRangeFilter;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.NumericUtils;
import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.ElasticSearchIllegalArgumentException;
@ -140,11 +143,10 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
private String nullValue; private String nullValue;
protected IpFieldMapper(Names names, int precisionStep, protected IpFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
float boost, FieldType fieldType,
String nullValue, Explicit<Boolean> ignoreMalformed, String nullValue, Explicit<Boolean> ignoreMalformed,
PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) { PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
super(names, precisionStep, null, boost, fieldType, super(names, precisionStep, boost, fieldType,
ignoreMalformed, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)), ignoreMalformed, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)),
new NamedAnalyzer("_ip/max", new NumericIpAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings); new NamedAnalyzer("_ip/max", new NumericIpAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings);
this.nullValue = nullValue; this.nullValue = nullValue;
@ -227,13 +229,6 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
true, true); true, true);
} }
@Override
public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) {
// Lucene 4 Upgrade: It's surprising this uses FuzzyQuery instead of NumericRangeQuery
int edits = FuzzyQuery.floatToEdits((float) minSim, value.codePointCount(0, value.length()));
return new FuzzyQuery(names.createIndexNameTerm(indexedValueForSearch(value)), edits, prefixLength, maxExpansions, transpositions);
}
@Override @Override
public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) { public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
return NumericRangeQuery.newLongRange(names.indexName(), precisionStep, return NumericRangeQuery.newLongRange(names.indexName(), precisionStep,

View File

@ -619,4 +619,35 @@ public class SimpleQueryTests extends AbstractNodesTests {
assertThat(searchResponse.hits().totalHits(), equalTo(2l)); assertThat(searchResponse.hits().totalHits(), equalTo(2l));
} }
@Test
public void testFuzzyQueryString() {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("index.number_of_shards", 1)).execute().actionGet();
client.prepareIndex("test", "type1", "1").setSource("str", "kimchy", "date", "2012-02-01", "num", 12).execute().actionGet();
client.prepareIndex("test", "type1", "2").setSource("str", "shay", "date", "2012-02-05", "num", 20).execute().actionGet();
client.admin().indices().prepareRefresh().execute().actionGet();
SearchResponse searchResponse = client.prepareSearch()
.setQuery(queryString("str:kimcy~1"))
.execute().actionGet();
assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0));
assertThat(searchResponse.hits().totalHits(), equalTo(1l));
assertThat(searchResponse.hits().getAt(0).id(), equalTo("1"));
searchResponse = client.prepareSearch()
.setQuery(queryString("num:11~1"))
.execute().actionGet();
assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0));
assertThat(searchResponse.hits().totalHits(), equalTo(1l));
assertThat(searchResponse.hits().getAt(0).id(), equalTo("1"));
// Note, this test fails, i.e returns 0 results, the reason is that Lucene QP only supports numbers after the ~
// once this is changed in lucene to support strings, then this test will fail (good!)
searchResponse = client.prepareSearch()
.setQuery(queryString("date:2012-02-02~1d"))
.execute().actionGet();
assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0));
assertThat(searchResponse.hits().totalHits(), equalTo(0l));
}
} }