From 6cec73c2010f98df1de8f3769abca49b3044ddfe Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 31 Jan 2013 12:22:52 +0100 Subject: [PATCH] remove fuzzy factor from mapping (internally implemented) we want to support ~ notion in query parser for types other than strings, we are getting there, one can do now age:10~5, we would love to support it for dates, as in timestamp:2012-10-10~5d, but that requires changes in the query parser to support strings after the ~ sign --- .../classic/MapperQueryParser.java | 55 ++++++++++++++++--- .../index/mapper/FieldMapper.java | 2 - .../mapper/core/AbstractFieldMapper.java | 6 -- .../index/mapper/core/ByteFieldMapper.java | 19 +------ .../index/mapper/core/DateFieldMapper.java | 32 +---------- .../index/mapper/core/DoubleFieldMapper.java | 20 +------ .../index/mapper/core/FloatFieldMapper.java | 19 +------ .../index/mapper/core/IntegerFieldMapper.java | 21 +------ .../index/mapper/core/LongFieldMapper.java | 20 +------ .../index/mapper/core/NumberFieldMapper.java | 29 +--------- .../index/mapper/core/ShortFieldMapper.java | 20 +------ .../index/mapper/core/TypeParsers.java | 2 - .../mapper/internal/BoostFieldMapper.java | 12 +--- .../mapper/internal/SizeFieldMapper.java | 2 +- .../index/mapper/internal/TTLFieldMapper.java | 2 +- .../mapper/internal/TimestampFieldMapper.java | 2 +- .../index/mapper/ip/IpFieldMapper.java | 17 ++---- .../search/query/SimpleQueryTests.java | 51 +++++++++++++---- 18 files changed, 121 insertions(+), 210 deletions(-) diff --git a/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java b/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java index b2c0d883fe0..f92ba2b82b2 100644 --- a/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java +++ b/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java @@ -128,6 +128,48 @@ public class MapperQueryParser extends QueryParser { this.analyzeWildcard = settings.analyzeWildcard(); } + /** + * We override this one so we can get the fuzzy part to be treated as string, so people can do: "age:10~5". Note, + * we would love to support also "timestamp:2012-10-10~5d", but sadly the parser expects only numbers after the ~, + * hopefully we can change that in Lucene. + */ + @Override + Query handleBareTokenQuery(String qfield, Token term, Token fuzzySlop, boolean prefix, boolean wildcard, boolean fuzzy, boolean regexp) throws ParseException { + Query q; + + String termImage = discardEscapeChar(term.image); + if (wildcard) { + q = getWildcardQuery(qfield, term.image); + } else if (prefix) { + q = getPrefixQuery(qfield, + discardEscapeChar(term.image.substring + (0, term.image.length() - 1))); + } else if (regexp) { + q = getRegexpQuery(qfield, term.image.substring(1, term.image.length() - 1)); + } else if (fuzzy) { +// float fms = fuzzyMinSim; +// try { +// fms = Float.valueOf(fuzzySlop.image.substring(1)).floatValue(); +// } catch (Exception ignored) { +// } +// if (fms < 0.0f) { +// throw new ParseException("Minimum similarity for a FuzzyQuery has to be between 0.0f and 1.0f !"); +// } else if (fms >= 1.0f && fms != (int) fms) { +// throw new ParseException("Fractional edit distances are not allowed!"); +// } +// q = getFuzzyQuery(qfield, termImage, fms); + if (fuzzySlop.image.length() == 1) { + q = getFuzzyQuery(qfield, termImage, Float.toString(fuzzyMinSim)); + } else { + q = getFuzzyQuery(qfield, termImage, fuzzySlop.image.substring(1)); + } + } else { + + q = getFieldQuery(qfield, termImage, false); + } + return q; + } + @Override protected Query newTermQuery(Term term) { if (currentMapper != null) { @@ -295,9 +337,9 @@ public class MapperQueryParser extends QueryParser { part2 = null; } if (lowercaseExpandedTerms) { - part1 = part1==null ? null : part1.toLowerCase(locale); - part2 = part2==null ? null : part2.toLowerCase(locale); - } + part1 = part1 == null ? null : part1.toLowerCase(locale); + part2 = part2 == null ? null : part2.toLowerCase(locale); + } Collection fields = extractMultiFields(field); if (fields != null) { if (fields.size() == 1) { @@ -356,8 +398,7 @@ public class MapperQueryParser extends QueryParser { return newRangeQuery(field, part1, part2, startInclusive, endInclusive); } - @Override - protected Query getFuzzyQuery(String field, String termStr, float minSimilarity) throws ParseException { + protected Query getFuzzyQuery(String field, String termStr, String minSimilarity) throws ParseException { if (lowercaseExpandedTerms) { termStr = termStr.toLowerCase(locale); } @@ -395,7 +436,7 @@ public class MapperQueryParser extends QueryParser { } } - private Query getFuzzyQuerySingle(String field, String termStr, float minSimilarity) throws ParseException { + private Query getFuzzyQuerySingle(String field, String termStr, String minSimilarity) throws ParseException { currentMapper = null; MapperService.SmartNameFieldMappers fieldMappers = parseContext.smartFieldMappers(field); if (fieldMappers != null) { @@ -413,7 +454,7 @@ public class MapperQueryParser extends QueryParser { } } } - return super.getFuzzyQuery(field, termStr, minSimilarity); + return super.getFuzzyQuery(field, termStr, Float.parseFloat(minSimilarity)); } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index e3a4ff96e8b..bb519b1928c 100644 --- a/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -177,8 +177,6 @@ public interface FieldMapper { Query fuzzyQuery(String value, String minSim, int prefixLength, int maxExpansions, boolean transpositions); - Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions); - Query prefixQuery(Object value, @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryParseContext context); Filter prefixFilter(Object value, @Nullable QueryParseContext context); diff --git a/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java index 88f7c57a8a7..cf6a320da74 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java @@ -476,12 +476,6 @@ public abstract class AbstractFieldMapper implements FieldMapper, Mapper { return new FuzzyQuery(names.createIndexNameTerm(indexedValueForSearch(value)), edits, prefixLength, maxExpansions, transpositions); } - @Override - public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) { - int edits = FuzzyQuery.floatToEdits((float) minSim, value.codePointCount(0, value.length())); - return new FuzzyQuery(names.createIndexNameTerm(indexedValueForSearch(value)), edits, prefixLength, maxExpansions, transpositions); - } - @Override public Query prefixQuery(Object value, @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryParseContext context) { PrefixQuery query = new PrefixQuery(names().createIndexNameTerm(indexedValueForSearch(value))); diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java index 4239a12523f..61e39690941 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java @@ -89,7 +89,7 @@ public class ByteFieldMapper extends NumberFieldMapper { public ByteFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); ByteFieldMapper fieldMapper = new ByteFieldMapper(buildNames(context), - precisionStep, fuzzyFactor, boost, fieldType, nullValue, ignoreMalformed(context), + precisionStep, boost, fieldType, nullValue, ignoreMalformed(context), provider, similarity, fieldDataSettings); fieldMapper.includeInAll(includeInAll); return fieldMapper; @@ -116,9 +116,9 @@ public class ByteFieldMapper extends NumberFieldMapper { private String nullValueAsString; - protected ByteFieldMapper(Names names, int precisionStep, String fuzzyFactor, float boost, FieldType fieldType, + protected ByteFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Byte nullValue, Explicit ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) { - super(names, precisionStep, fuzzyFactor, boost, fieldType, + super(names, precisionStep, boost, fieldType, ignoreMalformed, new NamedAnalyzer("_byte/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)), new NamedAnalyzer("_byte/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings); this.nullValue = nullValue; @@ -190,16 +190,6 @@ public class ByteFieldMapper extends NumberFieldMapper { true, true); } - @Override - public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) { - byte iValue = Byte.parseByte(value); - byte iSim = (byte) (minSim * dFuzzyFactor); - return NumericRangeQuery.newIntRange(names.indexName(), precisionStep, - iValue - iSim, - iValue + iSim, - true, true); - } - @Override public Query termQuery(Object value, @Nullable QueryParseContext context) { int iValue = parseValue(value); @@ -351,9 +341,6 @@ public class ByteFieldMapper extends NumberFieldMapper { if (precisionStep != Defaults.PRECISION_STEP) { builder.field("precision_step", precisionStep); } - if (fuzzyFactor != Defaults.FUZZY_FACTOR) { - builder.field("fuzzy_factor", fuzzyFactor); - } if (nullValue != null) { builder.field("null_value", nullValue); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index db22e55de44..0d2bbc4d471 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -116,7 +116,7 @@ public class DateFieldMapper extends NumberFieldMapper { } fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter, - precisionStep, fuzzyFactor, boost, fieldType, nullValue, + precisionStep, boost, fieldType, nullValue, timeUnit, parseUpperInclusive, ignoreMalformed(context), provider, similarity, fieldDataSettings); fieldMapper.includeInAll(includeInAll); return fieldMapper; @@ -153,11 +153,10 @@ public class DateFieldMapper extends NumberFieldMapper { protected final TimeUnit timeUnit; - protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter, int precisionStep, String fuzzyFactor, - float boost, FieldType fieldType, + protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter, int precisionStep, float boost, FieldType fieldType, String nullValue, TimeUnit timeUnit, boolean parseUpperInclusive, Explicit ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) { - super(names, precisionStep, fuzzyFactor, boost, fieldType, + super(names, precisionStep, boost, fieldType, ignoreMalformed, new NamedAnalyzer("_date/" + precisionStep, new NumericDateAnalyzer(precisionStep, dateTimeFormatter.parser())), new NamedAnalyzer("_date/max", new NumericDateAnalyzer(Integer.MAX_VALUE, dateTimeFormatter.parser())), @@ -179,18 +178,6 @@ public class DateFieldMapper extends NumberFieldMapper { return new FieldDataType("long"); } - @Override - protected double parseFuzzyFactor(String fuzzyFactor) { - if (fuzzyFactor == null) { - return 1.0d; - } - try { - return TimeValue.parseTimeValue(fuzzyFactor, null).millis(); - } catch (Exception e) { - return Double.parseDouble(fuzzyFactor); - } - } - @Override protected int maxPrecisionStep() { return 64; @@ -266,16 +253,6 @@ public class DateFieldMapper extends NumberFieldMapper { true, true); } - @Override - public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) { - long iValue = dateMathParser.parse(value, System.currentTimeMillis()); - long iSim = (long) (minSim * dFuzzyFactor); - return NumericRangeQuery.newLongRange(names.indexName(), precisionStep, - iValue - iSim, - iValue + iSim, - true, true); - } - @Override public Query termQuery(Object value, @Nullable QueryParseContext context) { long now = context == null ? System.currentTimeMillis() : context.nowInMillis(); @@ -426,9 +403,6 @@ public class DateFieldMapper extends NumberFieldMapper { if (precisionStep != Defaults.PRECISION_STEP) { builder.field("precision_step", precisionStep); } - if (fuzzyFactor != Defaults.FUZZY_FACTOR) { - builder.field("fuzzy_factor", fuzzyFactor); - } builder.field("format", dateTimeFormatter.format()); if (nullValue != null) { builder.field("null_value", nullValue); diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java index 7eedf0e09d8..49e0e60b02d 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java @@ -89,7 +89,7 @@ public class DoubleFieldMapper extends NumberFieldMapper { public DoubleFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); DoubleFieldMapper fieldMapper = new DoubleFieldMapper(buildNames(context), - precisionStep, fuzzyFactor, boost, fieldType, nullValue, + precisionStep, boost, fieldType, nullValue, ignoreMalformed(context), provider, similarity, fieldDataSettings); fieldMapper.includeInAll(includeInAll); return fieldMapper; @@ -117,11 +117,10 @@ public class DoubleFieldMapper extends NumberFieldMapper { private String nullValueAsString; - protected DoubleFieldMapper(Names names, int precisionStep, String fuzzyFactor, - float boost, FieldType fieldType, + protected DoubleFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Double nullValue, Explicit ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) { - super(names, precisionStep, fuzzyFactor, boost, fieldType, + super(names, precisionStep, boost, fieldType, ignoreMalformed, new NamedAnalyzer("_double/" + precisionStep, new NumericDoubleAnalyzer(precisionStep)), new NamedAnalyzer("_double/max", new NumericDoubleAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings); this.nullValue = nullValue; @@ -185,16 +184,6 @@ public class DoubleFieldMapper extends NumberFieldMapper { true, true); } - @Override - public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) { - double iValue = Double.parseDouble(value); - double iSim = minSim * dFuzzyFactor; - return NumericRangeQuery.newDoubleRange(names.indexName(), precisionStep, - iValue - iSim, - iValue + iSim, - true, true); - } - @Override public Query termQuery(Object value, @Nullable QueryParseContext context) { double dValue = parseValue(value); @@ -351,9 +340,6 @@ public class DoubleFieldMapper extends NumberFieldMapper { if (precisionStep != Defaults.PRECISION_STEP) { builder.field("precision_step", precisionStep); } - if (fuzzyFactor != Defaults.FUZZY_FACTOR) { - builder.field("fuzzy_factor", fuzzyFactor); - } if (nullValue != null) { builder.field("null_value", nullValue); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java index bd56399128b..de10210abde 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java @@ -90,7 +90,7 @@ public class FloatFieldMapper extends NumberFieldMapper { public FloatFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); FloatFieldMapper fieldMapper = new FloatFieldMapper(buildNames(context), - precisionStep, fuzzyFactor, boost, fieldType, nullValue, + precisionStep, boost, fieldType, nullValue, ignoreMalformed(context), provider, similarity, fieldDataSettings); fieldMapper.includeInAll(includeInAll); return fieldMapper; @@ -117,9 +117,9 @@ public class FloatFieldMapper extends NumberFieldMapper { private String nullValueAsString; - protected FloatFieldMapper(Names names, int precisionStep, String fuzzyFactor, float boost, FieldType fieldType, + protected FloatFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Float nullValue, Explicit ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) { - super(names, precisionStep, fuzzyFactor, boost, fieldType, + super(names, precisionStep, boost, fieldType, ignoreMalformed, new NamedAnalyzer("_float/" + precisionStep, new NumericFloatAnalyzer(precisionStep)), new NamedAnalyzer("_float/max", new NumericFloatAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings); this.nullValue = nullValue; @@ -183,16 +183,6 @@ public class FloatFieldMapper extends NumberFieldMapper { true, true); } - @Override - public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) { - float iValue = Float.parseFloat(value); - float iSim = (float) (minSim * dFuzzyFactor); - return NumericRangeQuery.newFloatRange(names.indexName(), precisionStep, - iValue - iSim, - iValue + iSim, - true, true); - } - @Override public Query termQuery(Object value, @Nullable QueryParseContext context) { float fValue = parseValue(value); @@ -346,9 +336,6 @@ public class FloatFieldMapper extends NumberFieldMapper { if (precisionStep != Defaults.PRECISION_STEP) { builder.field("precision_step", precisionStep); } - if (fuzzyFactor != Defaults.FUZZY_FACTOR) { - builder.field("fuzzy_factor", fuzzyFactor); - } if (nullValue != null) { builder.field("null_value", nullValue); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java index 51b5437b1d2..12acc577c8f 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java @@ -89,8 +89,7 @@ public class IntegerFieldMapper extends NumberFieldMapper { @Override public IntegerFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); - IntegerFieldMapper fieldMapper = new IntegerFieldMapper(buildNames(context), - precisionStep, fuzzyFactor, boost, fieldType, + IntegerFieldMapper fieldMapper = new IntegerFieldMapper(buildNames(context), precisionStep, boost, fieldType, nullValue, ignoreMalformed(context), provider, similarity, fieldDataSettings); fieldMapper.includeInAll(includeInAll); return fieldMapper; @@ -117,11 +116,10 @@ public class IntegerFieldMapper extends NumberFieldMapper { private String nullValueAsString; - protected IntegerFieldMapper(Names names, int precisionStep, String fuzzyFactor, - float boost, FieldType fieldType, + protected IntegerFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Integer nullValue, Explicit ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) { - super(names, precisionStep, fuzzyFactor, boost, fieldType, + super(names, precisionStep, boost, fieldType, ignoreMalformed, new NamedAnalyzer("_int/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)), new NamedAnalyzer("_int/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings); this.nullValue = nullValue; @@ -189,16 +187,6 @@ public class IntegerFieldMapper extends NumberFieldMapper { true, true); } - @Override - public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) { - int iValue = Integer.parseInt(value); - int iSim = (int) (minSim * dFuzzyFactor); - return NumericRangeQuery.newIntRange(names.indexName(), precisionStep, - iValue - iSim, - iValue + iSim, - true, true); - } - @Override public Query termQuery(Object value, @Nullable QueryParseContext context) { int iValue = parseValue(value); @@ -351,9 +339,6 @@ public class IntegerFieldMapper extends NumberFieldMapper { if (precisionStep != Defaults.PRECISION_STEP) { builder.field("precision_step", precisionStep); } - if (fuzzyFactor != Defaults.FUZZY_FACTOR) { - builder.field("fuzzy_factor", fuzzyFactor); - } if (nullValue != null) { builder.field("null_value", nullValue); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java index cfcbff4245c..0235f05b8dd 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java @@ -90,7 +90,7 @@ public class LongFieldMapper extends NumberFieldMapper { public LongFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); LongFieldMapper fieldMapper = new LongFieldMapper(buildNames(context), - precisionStep, fuzzyFactor, boost, fieldType, nullValue, + precisionStep, boost, fieldType, nullValue, ignoreMalformed(context), provider, similarity, fieldDataSettings); fieldMapper.includeInAll(includeInAll); return fieldMapper; @@ -117,11 +117,10 @@ public class LongFieldMapper extends NumberFieldMapper { private String nullValueAsString; - protected LongFieldMapper(Names names, int precisionStep, String fuzzyFactor, - float boost, FieldType fieldType, + protected LongFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Long nullValue, Explicit ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) { - super(names, precisionStep, fuzzyFactor, boost, fieldType, + super(names, precisionStep, boost, fieldType, ignoreMalformed, new NamedAnalyzer("_long/" + precisionStep, new NumericLongAnalyzer(precisionStep)), new NamedAnalyzer("_long/max", new NumericLongAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings); this.nullValue = nullValue; @@ -189,16 +188,6 @@ public class LongFieldMapper extends NumberFieldMapper { true, true); } - @Override - public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) { - long iValue = Long.parseLong(value); - long iSim = (long) (minSim * dFuzzyFactor); - return NumericRangeQuery.newLongRange(names.indexName(), precisionStep, - iValue - iSim, - iValue + iSim, - true, true); - } - @Override public Query termQuery(Object value, @Nullable QueryParseContext context) { long iValue = parseValue(value); @@ -350,9 +339,6 @@ public class LongFieldMapper extends NumberFieldMapper { if (precisionStep != Defaults.PRECISION_STEP) { builder.field("precision_step", precisionStep); } - if (fuzzyFactor != Defaults.FUZZY_FACTOR) { - builder.field("fuzzy_factor", fuzzyFactor); - } if (nullValue != null) { builder.field("null_value", nullValue); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java index 214e96d0f56..318bb46034c 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java @@ -59,7 +59,6 @@ public abstract class NumberFieldMapper extends AbstractFieldM FIELD_TYPE.freeze(); } - public static final String FUZZY_FACTOR = null; public static final Explicit IGNORE_MALFORMED = new Explicit(false, false); } @@ -67,8 +66,6 @@ public abstract class NumberFieldMapper extends AbstractFieldM protected int precisionStep = Defaults.PRECISION_STEP; - protected String fuzzyFactor = Defaults.FUZZY_FACTOR; - private Boolean ignoreMalformed; public Builder(String name, FieldType fieldType) { @@ -100,11 +97,6 @@ public abstract class NumberFieldMapper extends AbstractFieldM return builder; } - public T fuzzyFactor(String fuzzyFactor) { - this.fuzzyFactor = fuzzyFactor; - return builder; - } - public T ignoreMalformed(boolean ignoreMalformed) { this.ignoreMalformed = ignoreMalformed; return builder; @@ -123,10 +115,6 @@ public abstract class NumberFieldMapper extends AbstractFieldM protected int precisionStep; - protected String fuzzyFactor; - - protected double dFuzzyFactor; - protected Boolean includeInAll; protected Explicit ignoreMalformed; @@ -138,8 +126,7 @@ public abstract class NumberFieldMapper extends AbstractFieldM } }; - protected NumberFieldMapper(Names names, int precisionStep, @Nullable String fuzzyFactor, - float boost, FieldType fieldType, + protected NumberFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Explicit ignoreMalformed, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) { @@ -150,18 +137,9 @@ public abstract class NumberFieldMapper extends AbstractFieldM } else { this.precisionStep = precisionStep; } - this.fuzzyFactor = fuzzyFactor; - this.dFuzzyFactor = parseFuzzyFactor(fuzzyFactor); this.ignoreMalformed = ignoreMalformed; } - protected double parseFuzzyFactor(String fuzzyFactor) { - if (fuzzyFactor == null) { - return 1.0d; - } - return Double.parseDouble(fuzzyFactor); - } - @Override public void includeInAll(Boolean includeInAll) { if (includeInAll != null) { @@ -237,9 +215,6 @@ public abstract class NumberFieldMapper extends AbstractFieldM @Override public abstract Query fuzzyQuery(String value, String minSim, int prefixLength, int maxExpansions, boolean transpositions); - @Override - public abstract Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions); - /** * A range filter based on the field data cache. */ @@ -265,8 +240,6 @@ public abstract class NumberFieldMapper extends AbstractFieldM NumberFieldMapper nfmMergeWith = (NumberFieldMapper) mergeWith; this.precisionStep = nfmMergeWith.precisionStep; this.includeInAll = nfmMergeWith.includeInAll; - this.fuzzyFactor = nfmMergeWith.fuzzyFactor; - this.dFuzzyFactor = parseFuzzyFactor(nfmMergeWith.fuzzyFactor); if (nfmMergeWith.ignoreMalformed.explicit()) { this.ignoreMalformed = nfmMergeWith.ignoreMalformed; } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java index 0014d12633d..b5ba9157048 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java @@ -90,7 +90,7 @@ public class ShortFieldMapper extends NumberFieldMapper { public ShortFieldMapper build(BuilderContext context) { fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f); ShortFieldMapper fieldMapper = new ShortFieldMapper(buildNames(context), - precisionStep, fuzzyFactor, boost, fieldType, nullValue, + precisionStep, boost, fieldType, nullValue, ignoreMalformed(context), provider, similarity, fieldDataSettings); fieldMapper.includeInAll(includeInAll); return fieldMapper; @@ -117,11 +117,10 @@ public class ShortFieldMapper extends NumberFieldMapper { private String nullValueAsString; - protected ShortFieldMapper(Names names, int precisionStep, String fuzzyFactor, - float boost, FieldType fieldType, + protected ShortFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, Short nullValue, Explicit ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) { - super(names, precisionStep, fuzzyFactor, boost, fieldType, + super(names, precisionStep, boost, fieldType, ignoreMalformed, new NamedAnalyzer("_short/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)), new NamedAnalyzer("_short/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings); this.nullValue = nullValue; @@ -193,16 +192,6 @@ public class ShortFieldMapper extends NumberFieldMapper { true, true); } - @Override - public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) { - short iValue = Short.parseShort(value); - short iSim = (short) (minSim * dFuzzyFactor); - return NumericRangeQuery.newIntRange(names.indexName(), precisionStep, - iValue - iSim, - iValue + iSim, - true, true); - } - @Override public Query termQuery(Object value, @Nullable QueryParseContext context) { int iValue = parseValueAsInt(value); @@ -354,9 +343,6 @@ public class ShortFieldMapper extends NumberFieldMapper { if (precisionStep != Defaults.PRECISION_STEP) { builder.field("precision_step", precisionStep); } - if (fuzzyFactor != Defaults.FUZZY_FACTOR) { - builder.field("fuzzy_factor", fuzzyFactor); - } if (nullValue != null) { builder.field("null_value", nullValue); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java b/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java index a6729ff037c..94890363a4b 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java @@ -50,8 +50,6 @@ public class TypeParsers { Object propNode = entry.getValue(); if (propName.equals("precision_step")) { builder.precisionStep(nodeIntegerValue(propNode)); - } else if (propName.equals("fuzzy_factor")) { - builder.fuzzyFactor(propNode.toString()); } else if (propName.equals("ignore_malformed")) { builder.ignoreMalformed(nodeBooleanValue(propNode)); } else if (propName.equals("omit_norms")) { diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java index e6c47b04aba..9a381304306 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java @@ -122,7 +122,7 @@ public class BoostFieldMapper extends NumberFieldMapper implements Intern protected BoostFieldMapper(String name, String indexName, int precisionStep, float boost, FieldType fieldType, Float nullValue, PostingsFormatProvider provider, @Nullable Settings fieldDataSettings) { - super(new Names(name, indexName, indexName, name), precisionStep, null, boost, fieldType, + super(new Names(name, indexName, indexName, name), precisionStep, boost, fieldType, Defaults.IGNORE_MALFORMED, new NamedAnalyzer("_float/" + precisionStep, new NumericFloatAnalyzer(precisionStep)), new NamedAnalyzer("_float/max", new NumericFloatAnalyzer(Integer.MAX_VALUE)), provider, null, fieldDataSettings); this.nullValue = nullValue; @@ -185,16 +185,6 @@ public class BoostFieldMapper extends NumberFieldMapper implements Intern true, true); } - @Override - public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) { - float iValue = Float.parseFloat(value); - float iSim = (float) (minSim * dFuzzyFactor); - return NumericRangeQuery.newFloatRange(names.indexName(), precisionStep, - iValue - iSim, - iValue + iSim, - true, true); - } - @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) { return NumericRangeQuery.newFloatRange(names.indexName(), precisionStep, diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java index a19766a6dbd..656931906a3 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java @@ -101,7 +101,7 @@ public class SizeFieldMapper extends IntegerFieldMapper implements RootMapper { } public SizeFieldMapper(boolean enabled, FieldType fieldType, PostingsFormatProvider provider, @Nullable Settings fieldDataSettings) { - super(new Names(Defaults.NAME), Defaults.PRECISION_STEP, Defaults.FUZZY_FACTOR, + super(new Names(Defaults.NAME), Defaults.PRECISION_STEP, Defaults.BOOST, fieldType, Defaults.NULL_VALUE, Defaults.IGNORE_MALFORMED, provider, null, fieldDataSettings); this.enabled = enabled; } diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java index c68cda4dc71..1767b29ea86 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java @@ -120,7 +120,7 @@ public class TTLFieldMapper extends LongFieldMapper implements InternalMapper, R protected TTLFieldMapper(FieldType fieldType, boolean enabled, long defaultTTL, Explicit ignoreMalformed, PostingsFormatProvider provider, @Nullable Settings fieldDataSettings) { super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), Defaults.PRECISION_STEP, - Defaults.FUZZY_FACTOR, Defaults.BOOST, fieldType, Defaults.NULL_VALUE, ignoreMalformed, + Defaults.BOOST, fieldType, Defaults.NULL_VALUE, ignoreMalformed, provider, null, fieldDataSettings); this.enabled = enabled; this.defaultTTL = defaultTTL; diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java index f5e0b2dcd6d..29dce21efef 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java @@ -138,7 +138,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap FormatDateTimeFormatter dateTimeFormatter, boolean parseUpperInclusive, Explicit ignoreMalformed, PostingsFormatProvider provider, @Nullable Settings fieldDataSettings) { super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter, - Defaults.PRECISION_STEP, Defaults.FUZZY_FACTOR, Defaults.BOOST, fieldType, + Defaults.PRECISION_STEP, Defaults.BOOST, fieldType, Defaults.NULL_VALUE, TimeUnit.MILLISECONDS /*always milliseconds*/, parseUpperInclusive, ignoreMalformed, provider, null, fieldDataSettings); this.enabled = enabled; diff --git a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java index 0f062f411cf..e6b50c6a633 100644 --- a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java @@ -22,7 +22,10 @@ package org.elasticsearch.index.mapper.ip; import org.apache.lucene.analysis.NumericTokenStream; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.search.*; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.NumericRangeFilter; +import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.ElasticSearchIllegalArgumentException; @@ -140,11 +143,10 @@ public class IpFieldMapper extends NumberFieldMapper { private String nullValue; - protected IpFieldMapper(Names names, int precisionStep, - float boost, FieldType fieldType, + protected IpFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType, String nullValue, Explicit ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) { - super(names, precisionStep, null, boost, fieldType, + super(names, precisionStep, boost, fieldType, ignoreMalformed, new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)), new NamedAnalyzer("_ip/max", new NumericIpAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings); this.nullValue = nullValue; @@ -227,13 +229,6 @@ public class IpFieldMapper extends NumberFieldMapper { true, true); } - @Override - public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions, boolean transpositions) { - // Lucene 4 Upgrade: It's surprising this uses FuzzyQuery instead of NumericRangeQuery - int edits = FuzzyQuery.floatToEdits((float) minSim, value.codePointCount(0, value.length())); - return new FuzzyQuery(names.createIndexNameTerm(indexedValueForSearch(value)), edits, prefixLength, maxExpansions, transpositions); - } - @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) { return NumericRangeQuery.newLongRange(names.indexName(), precisionStep, diff --git a/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java b/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java index 160105c7e98..af018fc2e42 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java @@ -105,7 +105,7 @@ public class SimpleQueryTests extends AbstractNodesTests { assertTrue(e.getMessage().endsWith("IllegalStateException[field \"field1\" was indexed without position data; cannot run PhraseQuery (term=quick)]; }")); } } - + @Test public void testCommonTermsQuery() throws Exception { try { @@ -126,40 +126,40 @@ public class SimpleQueryTests extends AbstractNodesTests { assertThat(searchResponse.hits().totalHits(), equalTo(2l)); assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1")); assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2")); - - + + searchResponse = client.prepareSearch().setQuery(QueryBuilders.commonTerms("field1", "the quick brown").cutoffFrequency(3).lowFreqOperator(Operator.OR)).execute().actionGet(); assertThat(searchResponse.hits().totalHits(), equalTo(3l)); assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1")); assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2")); assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("3")); - + searchResponse = client.prepareSearch().setQuery(QueryBuilders.commonTerms("field1", "the quick brown").cutoffFrequency(3).analyzer("standard")).execute().actionGet(); assertThat(searchResponse.hits().totalHits(), equalTo(3l)); // standard drops "the" since its a stopword assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1")); assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("3")); assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("2")); - + // try the same with match query searchResponse = client.prepareSearch().setQuery(QueryBuilders.matchQuery("field1", "the quick brown").cutoffFrequency(3).operator(MatchQueryBuilder.Operator.AND)).execute().actionGet(); assertThat(searchResponse.hits().totalHits(), equalTo(2l)); assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1")); assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2")); - + searchResponse = client.prepareSearch().setQuery(QueryBuilders.matchQuery("field1", "the quick brown").cutoffFrequency(3).operator(MatchQueryBuilder.Operator.OR)).execute().actionGet(); assertThat(searchResponse.hits().totalHits(), equalTo(3l)); assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1")); assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2")); assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("3")); - + searchResponse = client.prepareSearch().setQuery(QueryBuilders.matchQuery("field1", "the quick brown").cutoffFrequency(3).operator(MatchQueryBuilder.Operator.AND).analyzer("standard")).execute().actionGet(); assertThat(searchResponse.hits().totalHits(), equalTo(3l)); // standard drops "the" since its a stopword assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1")); assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("3")); assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("2")); - + // try the same with multi match query searchResponse = client.prepareSearch().setQuery(QueryBuilders.multiMatchQuery("the quick brown", "field1", "field2").cutoffFrequency(3).operator(MatchQueryBuilder.Operator.AND)).execute().actionGet(); assertThat(searchResponse.hits().totalHits(), equalTo(3l)); @@ -167,7 +167,7 @@ public class SimpleQueryTests extends AbstractNodesTests { assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("1")); assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("2")); } - + @Test public void testOmitTermFreqsAndPositions() throws Exception { // backwards compat test! @@ -223,7 +223,7 @@ public class SimpleQueryTests extends AbstractNodesTests { searchResponse = client.prepareSearch().setQuery(queryString("v?l*e?1").analyzeWildcard(true)).execute().actionGet(); assertThat(searchResponse.hits().totalHits(), equalTo(1l)); } - + @Test public void testLowercaseExpandedTerms() { try { @@ -619,4 +619,35 @@ public class SimpleQueryTests extends AbstractNodesTests { assertThat(searchResponse.hits().totalHits(), equalTo(2l)); } + @Test + public void testFuzzyQueryString() { + client.admin().indices().prepareDelete().execute().actionGet(); + + client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("index.number_of_shards", 1)).execute().actionGet(); + client.prepareIndex("test", "type1", "1").setSource("str", "kimchy", "date", "2012-02-01", "num", 12).execute().actionGet(); + client.prepareIndex("test", "type1", "2").setSource("str", "shay", "date", "2012-02-05", "num", 20).execute().actionGet(); + client.admin().indices().prepareRefresh().execute().actionGet(); + + SearchResponse searchResponse = client.prepareSearch() + .setQuery(queryString("str:kimcy~1")) + .execute().actionGet(); + assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0)); + assertThat(searchResponse.hits().totalHits(), equalTo(1l)); + assertThat(searchResponse.hits().getAt(0).id(), equalTo("1")); + + searchResponse = client.prepareSearch() + .setQuery(queryString("num:11~1")) + .execute().actionGet(); + assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0)); + assertThat(searchResponse.hits().totalHits(), equalTo(1l)); + assertThat(searchResponse.hits().getAt(0).id(), equalTo("1")); + + // Note, this test fails, i.e returns 0 results, the reason is that Lucene QP only supports numbers after the ~ + // once this is changed in lucene to support strings, then this test will fail (good!) + searchResponse = client.prepareSearch() + .setQuery(queryString("date:2012-02-02~1d")) + .execute().actionGet(); + assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0)); + assertThat(searchResponse.hits().totalHits(), equalTo(0l)); + } }