From 6bd49091023d33d8410a8e2ee37d9dbbc9997279 Mon Sep 17 00:00:00 2001 From: Yonik Seeley Date: Mon, 23 Mar 2009 22:08:44 +0000 Subject: [PATCH] SOLR-939: ValueSourceRangeFilter/Query, frange parser git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@757570 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 2 + .../org/apache/solr/schema/DateField.java | 72 ++++- .../solr/schema/SortableDoubleField.java | 10 +- .../solr/schema/SortableFloatField.java | 10 +- .../apache/solr/schema/SortableIntField.java | 10 +- .../apache/solr/schema/SortableLongField.java | 10 +- src/java/org/apache/solr/schema/StrField.java | 67 +++++ .../solr/search/BoostQParserPlugin.java | 2 +- .../search/FunctionRangeQParserPlugin.java | 69 +++++ .../org/apache/solr/search/QParserPlugin.java | 1 + .../solr/search/function/DocValues.java | 76 ++++- .../search/function/DoubleFieldSource.java | 67 ++++- .../solr/search/function/IntFieldSource.java | 34 +++ .../solr/search/function/LongFieldSource.java | 41 ++- .../solr/search/function/OrdFieldSource.java | 22 +- .../search/function/StringIndexDocValues.java | 86 ++++++ .../solr/search/function/ValueSource.java | 60 ++++ .../function/ValueSourceRangeFilter.java | 94 ++++++ .../solr/util/AbstractSolrTestCase.java | 19 ++ .../apache/solr/search/TestRangeQuery.java | 277 ++++++++++++++++++ src/test/test-files/solr/conf/schema11.xml | 16 + 21 files changed, 1007 insertions(+), 38 deletions(-) create mode 100755 src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java create mode 100755 src/java/org/apache/solr/search/function/StringIndexDocValues.java create mode 100755 src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java create mode 100644 src/test/org/apache/solr/search/TestRangeQuery.java diff --git a/CHANGES.txt b/CHANGES.txt index 1dc582a10ce..02522db4aa1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -189,6 +189,8 @@ New Features 32. SOLR-844: A SolrServer implementation to front-end multiple solr servers and provides load balancing and failover support (Noble Paul, Mark Miller, hossman via shalin) +33. SOLR-939: ValueSourceRangeFilter/Query - filter based on values in a FieldCache entry or on any arbitrary function of field values. (yonik) + Optimizations ---------------------- diff --git a/src/java/org/apache/solr/schema/DateField.java b/src/java/org/apache/solr/schema/DateField.java index 5d120529a85..4a8b39eb2cb 100644 --- a/src/java/org/apache/solr/schema/DateField.java +++ b/src/java/org/apache/solr/schema/DateField.java @@ -22,8 +22,9 @@ import org.apache.solr.request.XMLWriter; import org.apache.solr.request.TextResponseWriter; import org.apache.lucene.document.Fieldable; import org.apache.lucene.search.SortField; -import org.apache.solr.search.function.ValueSource; -import org.apache.solr.search.function.OrdFieldSource; +import org.apache.lucene.index.IndexReader; +import org.apache.solr.search.function.*; +import org.apache.solr.search.QParser; import org.apache.solr.util.DateMathParser; import java.util.Map; @@ -330,5 +331,70 @@ public class DateField extends FieldType { return (DateFormat) proto.clone(); } } - + + @Override + public ValueSource getValueSource(SchemaField field, QParser parser) { + return new DateFieldSource(field.getName(), field.getType()); + } } + + + +class DateFieldSource extends FieldCacheSource { + // NOTE: this is bad for serialization... but we currently need the fieldType for toInternal() + FieldType ft; + + public DateFieldSource(String name, FieldType ft) { + super(name); + this.ft = ft; + } + + public String description() { + return "date(" + field + ')'; + } + + public DocValues getValues(IndexReader reader) throws IOException { + return new StringIndexDocValues(this, reader, field) { + protected String toTerm(String readableValue) { + // needed for frange queries to work properly + return ft.toInternal(readableValue); + } + + public float floatVal(int doc) { + return (float)intVal(doc); + } + + public int intVal(int doc) { + int ord=order[doc]; + return ord; + } + + public long longVal(int doc) { + return (long)intVal(doc); + } + + public double doubleVal(int doc) { + return (double)intVal(doc); + } + + public String strVal(int doc) { + int ord=order[doc]; + return ft.indexedToReadable(lookup[ord]); + } + + public String toString(int doc) { + return description() + '=' + intVal(doc); + } + }; + } + + public boolean equals(Object o) { + return o instanceof DateFieldSource + && super.equals(o); + } + + private static int hcode = DateFieldSource.class.hashCode(); + public int hashCode() { + return hcode + super.hashCode(); + }; +} \ No newline at end of file diff --git a/src/java/org/apache/solr/schema/SortableDoubleField.java b/src/java/org/apache/solr/schema/SortableDoubleField.java index 579c7a1f909..143428f5e81 100644 --- a/src/java/org/apache/solr/schema/SortableDoubleField.java +++ b/src/java/org/apache/solr/schema/SortableDoubleField.java @@ -22,6 +22,7 @@ import org.apache.lucene.search.FieldCache; import org.apache.solr.search.function.ValueSource; import org.apache.solr.search.function.FieldCacheSource; import org.apache.solr.search.function.DocValues; +import org.apache.solr.search.function.StringIndexDocValues; import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.IndexReader; import org.apache.solr.util.NumberUtils; @@ -93,12 +94,13 @@ class SortableDoubleFieldSource extends FieldCacheSource { } public DocValues getValues(IndexReader reader) throws IOException { - final FieldCache.StringIndex index = cache.getStringIndex(reader, field); - final int[] order = index.order; - final String[] lookup = index.lookup; final double def = defVal; - return new DocValues() { + return new StringIndexDocValues(this, reader, field) { + protected String toTerm(String readableValue) { + return NumberUtils.double2sortableStr(readableValue); + } + public float floatVal(int doc) { return (float)doubleVal(doc); } diff --git a/src/java/org/apache/solr/schema/SortableFloatField.java b/src/java/org/apache/solr/schema/SortableFloatField.java index 43841687d3a..1a379715570 100644 --- a/src/java/org/apache/solr/schema/SortableFloatField.java +++ b/src/java/org/apache/solr/schema/SortableFloatField.java @@ -22,6 +22,7 @@ import org.apache.lucene.search.FieldCache; import org.apache.solr.search.function.ValueSource; import org.apache.solr.search.function.FieldCacheSource; import org.apache.solr.search.function.DocValues; +import org.apache.solr.search.function.StringIndexDocValues; import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.IndexReader; import org.apache.solr.util.NumberUtils; @@ -93,12 +94,13 @@ class SortableFloatFieldSource extends FieldCacheSource { } public DocValues getValues(IndexReader reader) throws IOException { - final FieldCache.StringIndex index = cache.getStringIndex(reader, field); - final int[] order = index.order; - final String[] lookup = index.lookup; final float def = defVal; - return new DocValues() { + return new StringIndexDocValues(this, reader, field) { + protected String toTerm(String readableValue) { + return NumberUtils.float2sortableStr(readableValue); + } + public float floatVal(int doc) { int ord=order[doc]; return ord==0 ? def : NumberUtils.SortableStr2float(lookup[ord]); diff --git a/src/java/org/apache/solr/schema/SortableIntField.java b/src/java/org/apache/solr/schema/SortableIntField.java index b6deecd3ab7..fd8b766a477 100644 --- a/src/java/org/apache/solr/schema/SortableIntField.java +++ b/src/java/org/apache/solr/schema/SortableIntField.java @@ -22,6 +22,7 @@ import org.apache.lucene.search.FieldCache; import org.apache.solr.search.function.ValueSource; import org.apache.solr.search.function.FieldCacheSource; import org.apache.solr.search.function.DocValues; +import org.apache.solr.search.function.StringIndexDocValues; import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.IndexReader; import org.apache.solr.util.NumberUtils; @@ -97,12 +98,13 @@ class SortableIntFieldSource extends FieldCacheSource { } public DocValues getValues(IndexReader reader) throws IOException { - final FieldCache.StringIndex index = cache.getStringIndex(reader, field); - final int[] order = index.order; - final String[] lookup = index.lookup; final int def = defVal; - return new DocValues() { + return new StringIndexDocValues(this, reader, field) { + protected String toTerm(String readableValue) { + return NumberUtils.int2sortableStr(readableValue); + } + public float floatVal(int doc) { return (float)intVal(doc); } diff --git a/src/java/org/apache/solr/schema/SortableLongField.java b/src/java/org/apache/solr/schema/SortableLongField.java index 7411b96ca18..809aabb0383 100644 --- a/src/java/org/apache/solr/schema/SortableLongField.java +++ b/src/java/org/apache/solr/schema/SortableLongField.java @@ -22,6 +22,7 @@ import org.apache.lucene.search.FieldCache; import org.apache.solr.search.function.ValueSource; import org.apache.solr.search.function.FieldCacheSource; import org.apache.solr.search.function.DocValues; +import org.apache.solr.search.function.StringIndexDocValues; import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.IndexReader; import org.apache.solr.util.NumberUtils; @@ -94,12 +95,13 @@ class SortableLongFieldSource extends FieldCacheSource { } public DocValues getValues(IndexReader reader) throws IOException { - final FieldCache.StringIndex index = cache.getStringIndex(reader, field); - final int[] order = index.order; - final String[] lookup = index.lookup; final long def = defVal; - return new DocValues() { + return new StringIndexDocValues(this, reader, field) { + protected String toTerm(String readableValue) { + return NumberUtils.long2sortableStr(readableValue); + } + public float floatVal(int doc) { return (float)longVal(doc); } diff --git a/src/java/org/apache/solr/schema/StrField.java b/src/java/org/apache/solr/schema/StrField.java index ef5ec62d301..96e6cfa35bb 100644 --- a/src/java/org/apache/solr/schema/StrField.java +++ b/src/java/org/apache/solr/schema/StrField.java @@ -19,8 +19,15 @@ package org.apache.solr.schema; import org.apache.lucene.search.SortField; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.index.IndexReader; import org.apache.solr.request.XMLWriter; import org.apache.solr.request.TextResponseWriter; +import org.apache.solr.search.function.ValueSource; +import org.apache.solr.search.function.FieldCacheSource; +import org.apache.solr.search.function.DocValues; +import org.apache.solr.search.function.StringIndexDocValues; +import org.apache.solr.search.QParser; +import org.apache.solr.util.NumberUtils; import java.util.Map; import java.io.IOException; @@ -43,4 +50,64 @@ public class StrField extends CompressableField { public void write(TextResponseWriter writer, String name, Fieldable f) throws IOException { writer.writeStr(name, f.stringValue(), true); } + + public ValueSource getValueSource(SchemaField field, QParser parser) { + return super.getValueSource(field, parser); + } } + + +class StrFieldSource extends FieldCacheSource { + + public StrFieldSource(String field) { + super(field); + } + + public String description() { + return "str(" + field + ')'; + } + + public DocValues getValues(IndexReader reader) throws IOException { + return new StringIndexDocValues(this, reader, field) { + protected String toTerm(String readableValue) { + return readableValue; + } + + public float floatVal(int doc) { + return (float)intVal(doc); + } + + public int intVal(int doc) { + int ord=order[doc]; + return ord; + } + + public long longVal(int doc) { + return (long)intVal(doc); + } + + public double doubleVal(int doc) { + return (double)intVal(doc); + } + + public String strVal(int doc) { + int ord=order[doc]; + return lookup[ord]; + } + + public String toString(int doc) { + return description() + '=' + strVal(doc); + } + }; + } + + public boolean equals(Object o) { + return o instanceof StrFieldSource + && super.equals(o); + } + + private static int hcode = SortableFloatFieldSource.class.hashCode(); + public int hashCode() { + return hcode + super.hashCode(); + }; +} \ No newline at end of file diff --git a/src/java/org/apache/solr/search/BoostQParserPlugin.java b/src/java/org/apache/solr/search/BoostQParserPlugin.java index cb939cbffb3..9baef10fe33 100755 --- a/src/java/org/apache/solr/search/BoostQParserPlugin.java +++ b/src/java/org/apache/solr/search/BoostQParserPlugin.java @@ -65,7 +65,7 @@ public class BoostQParserPlugin extends QParserPlugin { public String[] getDefaultHighlightFields() { return baseParser.getDefaultHighlightFields(); } - + public Query getHighlightQuery() throws ParseException { return baseParser.getHighlightQuery(); } diff --git a/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java b/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java new file mode 100755 index 00000000000..fd357711982 --- /dev/null +++ b/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search; + +import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.search.function.*; + +/** + * Create a range query over a function. + *
Other parameters: + *
l, the lower bound, optional) + *
u, the upper bound, optional) + *
incl, include the lower bound: true/false, optional, default=true + *
incl, include the upper bound: true/false, optional, default=true + *
Example: {!frange l=1000 u=50000}myfield + */ +public class FunctionRangeQParserPlugin extends QParserPlugin { + public static String NAME = "frange"; + + public void init(NamedList args) { + } + + public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + return new QParser(qstr, localParams, params, req) { + ValueSource vs; + String funcStr; + + public Query parse() throws ParseException { + funcStr = localParams.get(QueryParsing.V, null); + Query funcQ = subQuery(funcStr, FunctionQParserPlugin.NAME).parse(); + if (funcQ instanceof FunctionQuery) { + vs = ((FunctionQuery)funcQ).getValueSource(); + } else { + vs = new QueryValueSource(funcQ, 0.0f); + } + + String l = localParams.get("l"); + String u = localParams.get("u"); + boolean includeLower = localParams.getBool("incl",true); + boolean includeUpper = localParams.getBool("incu",true); + + // TODO: add a score=val option to allow score to be the value + ValueSourceRangeFilter rf = new ValueSourceRangeFilter(vs, l, u, includeLower, includeUpper); + ConstantScoreQuery csq = new ConstantScoreQuery(rf); + return csq; + } + }; + } + +} diff --git a/src/java/org/apache/solr/search/QParserPlugin.java b/src/java/org/apache/solr/search/QParserPlugin.java index 2fd8f6eaec9..c52bab681fc 100755 --- a/src/java/org/apache/solr/search/QParserPlugin.java +++ b/src/java/org/apache/solr/search/QParserPlugin.java @@ -35,6 +35,7 @@ public abstract class QParserPlugin implements NamedListInitializedPlugin { FieldQParserPlugin.NAME, FieldQParserPlugin.class, RawQParserPlugin.NAME, RawQParserPlugin.class, NestedQParserPlugin.NAME, NestedQParserPlugin.class, + FunctionRangeQParserPlugin.NAME, FunctionRangeQParserPlugin.class, }; /** return a {@link QParser} */ diff --git a/src/java/org/apache/solr/search/function/DocValues.java b/src/java/org/apache/solr/search/function/DocValues.java index 20668045980..a005e4caa93 100644 --- a/src/java/org/apache/solr/search/function/DocValues.java +++ b/src/java/org/apache/solr/search/function/DocValues.java @@ -17,7 +17,11 @@ package org.apache.solr.search.function; -import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.*; +import org.apache.lucene.index.IndexReader; +import org.apache.solr.util.NumberUtils; + +import java.io.IOException; /** * Represents field values as different types. @@ -33,6 +37,7 @@ import org.apache.lucene.search.Explanation; // - For caching, Query objects are often used as keys... you don't // want the Query carrying around big objects public abstract class DocValues { + public byte byteVal(int doc) { throw new UnsupportedOperationException(); } public short shortVal(int doc) { throw new UnsupportedOperationException(); } @@ -42,7 +47,76 @@ public abstract class DocValues { public double doubleVal(int doc) { throw new UnsupportedOperationException(); } public String strVal(int doc) { throw new UnsupportedOperationException(); } public abstract String toString(int doc); + + public Explanation explain(int doc) { return new Explanation(floatVal(doc), toString(doc)); } + + public ValueSourceScorer getScorer(IndexReader reader) { + return new ValueSourceScorer(reader, this); + } + + // A RangeValueSource can't easily be a ValueSource that takes another ValueSource + // because it needs different behavior depending on the type of fields. There is also + // a setup cost - parsing and normalizing params, and doing a binary search on the StringIndex. + + public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) { + float lower; + float upper; + + if (lowerVal == null) { + lower = Float.NEGATIVE_INFINITY; + } else { + lower = Float.parseFloat(lowerVal); + } + if (upperVal == null) { + upper = Float.POSITIVE_INFINITY; + } else { + upper = Float.parseFloat(upperVal); + } + + final float l = lower; + final float u = upper; + + if (includeLower && includeUpper) { + return new ValueSourceScorer(reader, this) { + @Override + public boolean matchesValue(int doc) { + float docVal = floatVal(doc); + return docVal >= l && docVal <= u; + } + }; + } + else if (includeLower && !includeUpper) { + return new ValueSourceScorer(reader, this) { + @Override + public boolean matchesValue(int doc) { + float docVal = floatVal(doc); + return docVal >= l && docVal < u; + } + }; + } + else if (!includeLower && includeUpper) { + return new ValueSourceScorer(reader, this) { + @Override + public boolean matchesValue(int doc) { + float docVal = floatVal(doc); + return docVal > l && docVal <= u; + } + }; + } + else { + return new ValueSourceScorer(reader, this) { + @Override + public boolean matchesValue(int doc) { + float docVal = floatVal(doc); + return docVal > l && docVal < u; + } + }; + } + } } + + + diff --git a/src/java/org/apache/solr/search/function/DoubleFieldSource.java b/src/java/org/apache/solr/search/function/DoubleFieldSource.java index 9cfd6f9e2ca..e308c704184 100644 --- a/src/java/org/apache/solr/search/function/DoubleFieldSource.java +++ b/src/java/org/apache/solr/search/function/DoubleFieldSource.java @@ -74,7 +74,68 @@ public class DoubleFieldSource extends FieldCacheSource { public String toString(int doc) { return description() + '=' + floatVal(doc); } - }; + + @Override + public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) { + double lower,upper; + + if (lowerVal==null) { + lower = Double.NEGATIVE_INFINITY; + } else { + lower = Double.parseDouble(lowerVal); + } + + if (upperVal==null) { + upper = Double.POSITIVE_INFINITY; + } else { + upper = Double.parseDouble(upperVal); + } + + final double l = lower; + final double u = upper; + + + if (includeLower && includeUpper) { + return new ValueSourceScorer(reader, this) { + @Override + public boolean matchesValue(int doc) { + double docVal = doubleVal(doc); + return docVal >= l && docVal <= u; + } + }; + } + else if (includeLower && !includeUpper) { + return new ValueSourceScorer(reader, this) { + @Override + public boolean matchesValue(int doc) { + double docVal = doubleVal(doc); + return docVal >= l && docVal < u; + } + }; + } + else if (!includeLower && includeUpper) { + return new ValueSourceScorer(reader, this) { + @Override + public boolean matchesValue(int doc) { + double docVal = doubleVal(doc); + return docVal > l && docVal <= u; + } + }; + } + else { + return new ValueSourceScorer(reader, this) { + @Override + public boolean matchesValue(int doc) { + double docVal = doubleVal(doc); + return docVal > l && docVal < u; + } + }; + } + } + + + }; + } public boolean equals(Object o) { @@ -86,11 +147,9 @@ public class DoubleFieldSource extends FieldCacheSource { } public int hashCode() { - int h = parser == null ? Float.class.hashCode() : parser.getClass().hashCode(); + int h = parser == null ? Double.class.hashCode() : parser.getClass().hashCode(); h += super.hashCode(); return h; } - ; - } \ No newline at end of file diff --git a/src/java/org/apache/solr/search/function/IntFieldSource.java b/src/java/org/apache/solr/search/function/IntFieldSource.java index 4a4aec5aaf3..52fefaf6f46 100644 --- a/src/java/org/apache/solr/search/function/IntFieldSource.java +++ b/src/java/org/apache/solr/search/function/IntFieldSource.java @@ -46,6 +46,7 @@ public class IntFieldSource extends FieldCacheSource { return "int(" + field + ')'; } + public DocValues getValues(IndexReader reader) throws IOException { final int[] arr = (parser==null) ? cache.getInts(reader, field) : @@ -75,6 +76,39 @@ public class IntFieldSource extends FieldCacheSource { return description() + '=' + intVal(doc); } + @Override + public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) { + int lower,upper; + + // instead of using separate comparison functions, adjust the endpoints. + + if (lowerVal==null) { + lower = Integer.MIN_VALUE; + } else { + lower = Integer.parseInt(lowerVal); + if (!includeLower && lower < Integer.MAX_VALUE) lower++; + } + + if (upperVal==null) { + upper = Integer.MAX_VALUE; + } else { + upper = Integer.parseInt(upperVal); + if (!includeUpper && upper > Integer.MIN_VALUE) upper--; + } + + final int ll = lower; + final int uu = upper; + + return new ValueSourceScorer(reader, this) { + @Override + public boolean matchesValue(int doc) { + int val = arr[doc]; + // only check for deleted if it's the default value + // if (val==0 && reader.isDeleted(doc)) return false; + return val >= ll && val <= uu; + } + }; + } }; } diff --git a/src/java/org/apache/solr/search/function/LongFieldSource.java b/src/java/org/apache/solr/search/function/LongFieldSource.java index 2ad7d19f0a6..f600406b124 100644 --- a/src/java/org/apache/solr/search/function/LongFieldSource.java +++ b/src/java/org/apache/solr/search/function/LongFieldSource.java @@ -75,6 +75,43 @@ public class LongFieldSource extends FieldCacheSource { public String toString(int doc) { return description() + '=' + floatVal(doc); } + + + @Override + public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) { + long lower,upper; + + // instead of using separate comparison functions, adjust the endpoints. + + if (lowerVal==null) { + lower = Long.MIN_VALUE; + } else { + lower = Long.parseLong(lowerVal); + if (!includeLower && lower < Long.MAX_VALUE) lower++; + } + + if (upperVal==null) { + upper = Long.MAX_VALUE; + } else { + upper = Long.parseLong(upperVal); + if (!includeUpper && upper > Long.MIN_VALUE) upper--; + } + + final long ll = lower; + final long uu = upper; + + return new ValueSourceScorer(reader, this) { + @Override + public boolean matchesValue(int doc) { + long val = arr[doc]; + // only check for deleted if it's the default value + // if (val==0 && reader.isDeleted(doc)) return false; + return val >= ll && val <= uu; + } + }; + } + + }; } @@ -87,11 +124,9 @@ public class LongFieldSource extends FieldCacheSource { } public int hashCode() { - int h = parser == null ? Float.class.hashCode() : parser.getClass().hashCode(); + int h = parser == null ? Long.class.hashCode() : parser.getClass().hashCode(); h += super.hashCode(); return h; } - ; - } \ No newline at end of file diff --git a/src/java/org/apache/solr/search/function/OrdFieldSource.java b/src/java/org/apache/solr/search/function/OrdFieldSource.java index 45859d84ce4..1e0948a06d6 100644 --- a/src/java/org/apache/solr/search/function/OrdFieldSource.java +++ b/src/java/org/apache/solr/search/function/OrdFieldSource.java @@ -50,28 +50,32 @@ public class OrdFieldSource extends ValueSource { return "ord(" + field + ')'; } + public DocValues getValues(IndexReader reader) throws IOException { - final int[] arr = FieldCache.DEFAULT.getStringIndex(reader, field).order; - return new DocValues() { + return new StringIndexDocValues(this, reader, field) { + protected String toTerm(String readableValue) { + return readableValue; + } + public float floatVal(int doc) { - return (float)arr[doc]; + return (float)order[doc]; } public int intVal(int doc) { - return (int)arr[doc]; + return order[doc]; } public long longVal(int doc) { - return (long)arr[doc]; + return (long)order[doc]; } public double doubleVal(int doc) { - return (double)arr[doc]; + return (double)order[doc]; } public String strVal(int doc) { // the string value of the ordinal, not the string itself - return Integer.toString(arr[doc]); + return Integer.toString(order[doc]); } public String toString(int doc) { @@ -81,9 +85,7 @@ public class OrdFieldSource extends ValueSource { } public boolean equals(Object o) { - if (o.getClass() != OrdFieldSource.class) return false; - OrdFieldSource other = (OrdFieldSource)o; - return this.field.equals(field); + return o.getClass() == OrdFieldSource.class && this.field.equals(field); } private static final int hcode = OrdFieldSource.class.hashCode(); diff --git a/src/java/org/apache/solr/search/function/StringIndexDocValues.java b/src/java/org/apache/solr/search/function/StringIndexDocValues.java new file mode 100755 index 00000000000..1e7b4b71aba --- /dev/null +++ b/src/java/org/apache/solr/search/function/StringIndexDocValues.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.function; + +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.search.ExtendedFieldCache; +import org.apache.lucene.index.IndexReader; + +import java.io.IOException; + +/** Internal class, subject to change. + * Serves as base class for DocValues based on StringIndex + **/ +public abstract class StringIndexDocValues extends DocValues { + protected final FieldCache.StringIndex index; + protected final int[] order; + protected final String[] lookup; + protected final ValueSource vs; + + public StringIndexDocValues(ValueSource vs, IndexReader reader, String field) throws IOException { + index = ExtendedFieldCache.EXT_DEFAULT.getStringIndex(reader, field); + order = index.order; + lookup = index.lookup; + this.vs = vs; + } + + protected abstract String toTerm(String readableValue); + + @Override + public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) { + // TODO: are lowerVal and upperVal in indexed form or not? + lowerVal = lowerVal == null ? null : toTerm(lowerVal); + upperVal = upperVal == null ? null : toTerm(upperVal); + + int lower = Integer.MIN_VALUE; + if (lowerVal != null) { + lower = index.binarySearchLookup(lowerVal); + if (lower < 0) { + lower = -lower-1; + } else if (!includeLower) { + lower++; + } + } + + int upper = Integer.MAX_VALUE; + if (upperVal != null) { + upper = index.binarySearchLookup(upperVal); + if (upper < 0) { + upper = -upper-2; + } else if (!includeUpper) { + upper--; + } + } + + final int ll = lower; + final int uu = upper; + + return new ValueSourceScorer(reader, this) { + @Override + public boolean matchesValue(int doc) { + int ord = order[doc]; + return ord >= ll && ord <= uu; + } + }; + } + + public String toString(int doc) { + return vs.description() + '=' + strVal(doc); + } + + } diff --git a/src/java/org/apache/solr/search/function/ValueSource.java b/src/java/org/apache/solr/search/function/ValueSource.java index 147ef981f65..1fd7f068003 100644 --- a/src/java/org/apache/solr/search/function/ValueSource.java +++ b/src/java/org/apache/solr/search/function/ValueSource.java @@ -18,6 +18,7 @@ package org.apache.solr.search.function; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.*; import org.apache.solr.search.function.DocValues; import java.io.IOException; @@ -46,3 +47,62 @@ public abstract class ValueSource implements Serializable { } } + + +class ValueSourceScorer extends Scorer { + protected IndexReader reader; + private int doc = -1; + protected final int maxDoc; + protected final DocValues values; + protected boolean checkDeletes; + + protected ValueSourceScorer(IndexReader reader, DocValues values) { + super(null); + this.reader = reader; + this.maxDoc = reader.maxDoc(); + this.values = values; + setCheckDeletes(true); + } + + public IndexReader getReader() { return reader; } + + public void setCheckDeletes(boolean checkDeletes) { + this.checkDeletes = checkDeletes && reader.hasDeletions(); + } + + public boolean matches(int doc) { + return (!checkDeletes || !reader.isDeleted(maxDoc)) && matchesValue(doc); + } + + public boolean matchesValue(int doc) { + return true; + } + + public int doc() { + return doc; + } + + public boolean next() { + for(;;) { + doc++; + if (doc >= maxDoc) return false; + if (matches(doc)) return true; + } + } + + public boolean skipTo(int target) { + doc = target-1; + return next(); + } + + + public float score() throws IOException { + return values.floatVal(doc); + } + + public Explanation explain(int doc) throws IOException { + return values.explain(doc); + } +} + + diff --git a/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java b/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java new file mode 100755 index 00000000000..1fb7c2986d6 --- /dev/null +++ b/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.function; + +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.index.IndexReader; + +import java.io.IOException; + + +/** + * RangeFilter over a ValueSource. + */ +public class ValueSourceRangeFilter extends Filter { + private final ValueSource valueSource; + private final String lowerVal; + private final String upperVal; + private final boolean includeLower; + private final boolean includeUpper; + + public ValueSourceRangeFilter(ValueSource valueSource, + String lowerVal, + String upperVal, + boolean includeLower, + boolean includeUpper) { + this.valueSource = valueSource; + this.lowerVal = lowerVal; + this.upperVal = upperVal; + this.includeLower = lowerVal != null && includeLower; + this.includeUpper = upperVal != null && includeUpper; + } + + public DocIdSet getDocIdSet(final IndexReader reader) throws IOException { + return new DocIdSet() { + public DocIdSetIterator iterator() throws IOException { + return valueSource.getValues(reader).getRangeScorer(reader, lowerVal, upperVal, includeLower, includeUpper); + } + }; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("frange("); + sb.append(valueSource); + sb.append("):"); + sb.append(includeLower ? '[' : '{'); + sb.append(lowerVal == null ? "*" : lowerVal); + sb.append(" TO "); + sb.append(upperVal == null ? "*" : upperVal); + sb.append(includeUpper ? ']' : '}'); + return sb.toString(); + } + + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof ValueSourceRangeFilter)) return false; + ValueSourceRangeFilter other = (ValueSourceRangeFilter)o; + + if (!this.valueSource.equals(other.valueSource) + || this.includeLower != other.includeLower + || this.includeUpper != other.includeUpper + ) { return false; } + if (this.lowerVal != null ? !this.lowerVal.equals(other.lowerVal) : other.lowerVal != null) return false; + if (this.upperVal != null ? !this.upperVal.equals(other.upperVal) : other.upperVal != null) return false; + return true; + } + + public int hashCode() { + int h = valueSource.hashCode(); + h += lowerVal != null ? lowerVal.hashCode() : 0x572353db; + h = (h << 16) | (h >>> 16); // rotate to distinguish lower from upper + h += (upperVal != null ? (upperVal.hashCode()) : 0xe16fe9e7); + h += (includeLower ? 0xdaa47978 : 0) + + (includeUpper ? 0x9e634b57 : 0); + return h; + } +} diff --git a/src/java/org/apache/solr/util/AbstractSolrTestCase.java b/src/java/org/apache/solr/util/AbstractSolrTestCase.java index 08d1ba4a6f2..eafed7fcf75 100644 --- a/src/java/org/apache/solr/util/AbstractSolrTestCase.java +++ b/src/java/org/apache/solr/util/AbstractSolrTestCase.java @@ -21,6 +21,8 @@ package org.apache.solr.util; import org.apache.solr.core.SolrConfig; import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.SolrInputField; import org.apache.solr.common.util.XML; import org.apache.solr.request.*; import org.apache.solr.util.TestHarness; @@ -30,6 +32,8 @@ import junit.framework.TestCase; import javax.xml.xpath.XPathExpressionException; import java.io.*; +import java.util.List; +import java.util.ArrayList; /** * An Abstract base class that makes writing Solr JUnit tests "easier" @@ -222,6 +226,21 @@ public abstract class AbstractSolrTestCase extends TestCase { Doc d = doc(fieldsAndValues); return add(d); } + + /** + * Generates a simple <add><doc>... XML String with no options + */ + public String adoc(SolrInputDocument sdoc) { + List fields = new ArrayList(); + for (SolrInputField sf : sdoc) { + for (Object o : sf.getValues()) { + fields.add(sf.getName()); + fields.add(o.toString()); + } + } + return adoc(fields.toArray(new String[fields.size()])); + } + /** * Generates an <add><doc>... XML String with options diff --git a/src/test/org/apache/solr/search/TestRangeQuery.java b/src/test/org/apache/solr/search/TestRangeQuery.java new file mode 100644 index 00000000000..c87270a54dc --- /dev/null +++ b/src/test/org/apache/solr/search/TestRangeQuery.java @@ -0,0 +1,277 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search; + +import org.apache.solr.util.AbstractSolrTestCase; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.request.SolrQueryResponse; + +import java.util.*; + +import junit.framework.TestCase; + +public class TestRangeQuery extends AbstractSolrTestCase { + + public String getSchemaFile() { return "schema11.xml"; } + public String getSolrConfigFile() { return "solrconfig.xml"; } + public String getCoreName() { return "basic"; } + + + public void setUp() throws Exception { + // if you override setUp or tearDown, you better call + // the super classes version + super.setUp(); + } + public void tearDown() throws Exception { + // if you override setUp or tearDown, you better call + // the super classes version + super.tearDown(); + } + + Random r = new Random(1); + + void addInt(SolrInputDocument doc, int l, int u, String... fields) { + int v=0; + if (0==l && l==u) { + v=r.nextInt(); + } else { + v=r.nextInt(u-l)+l; + } + for (String field : fields) { + doc.addField(field, v); + } + } + + interface DocProcessor { + public void process(SolrInputDocument doc); + } + + public void createIndex(int nDocs, DocProcessor proc) { + for (int i=0; i norm_fields = new HashMap(); + norm_fields.put("foo_i", ints); + norm_fields.put("foo_l", longs); + norm_fields.put("foo_d", doubles); + + norm_fields.put("foo_ti", ints); + norm_fields.put("foo_tl", longs); + norm_fields.put("foo_td", doubles); + + norm_fields.put("foo_s", strings); + norm_fields.put("foo_dt", dates); + + + // fields that frange queries should work on + Map frange_fields = new HashMap(); + frange_fields.put("foo_i", ints); + frange_fields.put("foo_l", longs); + frange_fields.put("foo_d", doubles); + + frange_fields.put("foo_pi", ints); + frange_fields.put("foo_pl", longs); + frange_fields.put("foo_pd", doubles); + + frange_fields.put("foo_s", strings); + frange_fields.put("foo_dt", dates); + + Map all_fields = new HashMap(); + all_fields.putAll(norm_fields); + all_fields.putAll(frange_fields); + + for (int j=0; j fields = new ArrayList(); + fields.add("id"); + fields.add(""+j); + for (Map.Entry entry : all_fields.entrySet()) { + fields.add(entry.getKey()); + fields.add(entry.getValue()[j]); + } + assertU(adoc(fields.toArray(new String[fields.size()]))); + } + + assertU(commit()); + + // simple test of a function rather than just the field + assertQ(req("{!frange l=0 u=2}id"), "*[count(//doc)=3]"); + assertQ(req("{!frange l=0 u=2}product(id,2)"), "*[count(//doc)=2]"); + assertQ(req("{!frange l=100 u=102}sum(id,100)"), "*[count(//doc)=3]"); + + + for (Map.Entry entry : norm_fields.entrySet()) { + String f = entry.getKey(); + String[] v = entry.getValue(); + + assertQ(req(f + ":[* TO *]" ), "*[count(//doc)=3]"); + assertQ(req(f + ":["+v[0]+" TO "+v[2]+"]"), "*[count(//doc)=3]"); + assertQ(req(f + ":["+v[1]+" TO "+v[2]+"]"), "*[count(//doc)=2]"); + assertQ(req(f + ":["+v[0]+" TO "+v[1]+"]"), "*[count(//doc)=2]"); + assertQ(req(f + ":["+v[0]+" TO "+v[0]+"]"), "*[count(//doc)=1]"); + assertQ(req(f + ":["+v[1]+" TO "+v[1]+"]"), "*[count(//doc)=1]"); + assertQ(req(f + ":["+v[2]+" TO "+v[2]+"]"), "*[count(//doc)=1]"); + assertQ(req(f + ":["+v[3]+" TO "+v[3]+"]"), "*[count(//doc)=0]"); + assertQ(req(f + ":["+v[4]+" TO "+v[4]+"]"), "*[count(//doc)=0]"); + + assertQ(req(f + ":{"+v[0]+" TO "+v[2]+"}"), "*[count(//doc)=1]"); + assertQ(req(f + ":{"+v[1]+" TO "+v[2]+"}"), "*[count(//doc)=0]"); + assertQ(req(f + ":{"+v[0]+" TO "+v[1]+"}"), "*[count(//doc)=0]"); + assertQ(req(f + ":{"+v[3]+" TO "+v[4]+"}"), "*[count(//doc)=3]"); + } + + for (Map.Entry entry : frange_fields.entrySet()) { + String f = entry.getKey(); + String[] v = entry.getValue(); + + assertQ(req("{!frange}"+f ), "*[count(//doc)=3]"); + assertQ(req("{!frange" + " l="+v[0]+"}"+f ), "*[count(//doc)=3]"); + assertQ(req("{!frange" + " l="+v[1]+"}"+f ), "*[count(//doc)=2]"); + assertQ(req("{!frange" + " l="+v[2]+"}"+f ), "*[count(//doc)=1]"); + assertQ(req("{!frange" + " l="+v[3]+"}"+f ), "*[count(//doc)=3]"); + assertQ(req("{!frange" + " l="+v[4]+"}"+f ), "*[count(//doc)=0]"); + + assertQ(req("{!frange" + " u="+v[0]+"}"+f ), "*[count(//doc)=1]"); + assertQ(req("{!frange" + " u="+v[1]+"}"+f ), "*[count(//doc)=2]"); + assertQ(req("{!frange" + " u="+v[2]+"}"+f ), "*[count(//doc)=3]"); + assertQ(req("{!frange" + " u="+v[3]+"}"+f ), "*[count(//doc)=0]"); + assertQ(req("{!frange" + " u="+v[4]+"}"+f ), "*[count(//doc)=3]"); + + assertQ(req("{!frange incl=false" + " l="+v[0]+"}"+f ), "*[count(//doc)=2]"); + assertQ(req("{!frange incl=false" + " l="+v[1]+"}"+f ), "*[count(//doc)=1]"); + assertQ(req("{!frange incl=false" + " l="+v[2]+"}"+f ), "*[count(//doc)=0]"); + assertQ(req("{!frange incl=false" + " l="+v[3]+"}"+f ), "*[count(//doc)=3]"); + assertQ(req("{!frange incl=false" + " l="+v[4]+"}"+f ), "*[count(//doc)=0]"); + + assertQ(req("{!frange incu=false" + " u="+v[0]+"}"+f ), "*[count(//doc)=0]"); + assertQ(req("{!frange incu=false" + " u="+v[1]+"}"+f ), "*[count(//doc)=1]"); + assertQ(req("{!frange incu=false" + " u="+v[2]+"}"+f ), "*[count(//doc)=2]"); + assertQ(req("{!frange incu=false" + " u="+v[3]+"}"+f ), "*[count(//doc)=0]"); + assertQ(req("{!frange incu=false" + " u="+v[4]+"}"+f ), "*[count(//doc)=3]"); + + assertQ(req("{!frange incl=true incu=true" + " l=" +v[0] +" u="+v[2]+"}"+f ), "*[count(//doc)=3]"); + assertQ(req("{!frange incl=false incu=false" + " l=" +v[0] +" u="+v[2]+"}"+f ), "*[count(//doc)=1]"); + assertQ(req("{!frange incl=false incu=false" + " l=" +v[3] +" u="+v[4]+"}"+f ), "*[count(//doc)=3]"); + } + + } + + public void testRandomRangeQueries() throws Exception { + String handler=""; + final String[] fields = {"foo_s","foo_i","foo_l","foo_f","foo_d" // SortableIntField, etc + ,"foo_pi","foo_pl","foo_pf","foo_pd" // plain int IntField, etc + ,"foo_ti","foo_tl","foo_tf","foo_td" // trie numer fields + }; + final int l=5; + final int u=25; + + + createIndex(15, new DocProcessor() { + public void process(SolrInputDocument doc) { + addInt(doc, l,u, fields); + } + }); + assertU(commit()); + + // fields that a normal range query will work correctly on + String[] norm_fields = { + "foo_i","foo_l","foo_f","foo_d" + ,"foo_ti","foo_tl","foo_tf","foo_td" + + }; + + // fields that a value source range query should work on + String[] frange_fields = {"foo_i","foo_l","foo_f","foo_d", + "foo_pi","foo_pl","foo_pf","foo_pd"}; + + for (int i=0; i<1000; i++) { + int lower = l + r.nextInt(u-l+10)-5; + int upper = lower + r.nextInt(u+5-lower); + boolean lowerMissing = r.nextInt(10)==1; + boolean upperMissing = r.nextInt(10)==1; + boolean inclusive = lowerMissing || upperMissing || r.nextBoolean(); + + // lower=2; upper=2; inclusive=true; + // inclusive=true; lowerMissing=true; upperMissing=true; + + List qs = new ArrayList(); + for (String field : norm_fields) { + String q = field + ':' + (inclusive?'[':'{') + + (lowerMissing?"*":lower) + + " TO " + + (upperMissing?"*":upper) + + (inclusive?']':'}'); + qs.add(q); + } + for (String field : frange_fields) { + String q = "{!frange v="+field + + (lowerMissing?"":(" l="+lower)) + + (upperMissing?"":(" u="+upper)) + + (inclusive?"":" incl=false") + + (inclusive?"":" incu=false") + + "}"; + qs.add(q); + } + + SolrQueryResponse last=null; + for (String q : qs) { + // System.out.println("QUERY="+q); + SolrQueryResponse qr = h.queryAndResponse(handler, req("q",q,"rows","1000")); + if (last != null) { + // we only test if the same docs matched since some queries will include factors like idf, etc. + sameDocs((DocSet)qr.getValues().get("response"), (DocSet)last.getValues().get("response")); + } + last = qr; + } + } + } + + static boolean sameDocs(DocSet a, DocSet b) { + DocIterator i = a.iterator(); + // System.out.println("SIZES="+a.size() + "," + b.size()); + assertEquals(a.size(), b.size()); + while (i.hasNext()) { + int doc = i.nextDoc(); + if (!b.exists(doc)) { + TestCase.fail("Missing doc " + doc); + } + // System.out.println("MATCH! " + doc); + } + return true; + } +} \ No newline at end of file diff --git a/src/test/test-files/solr/conf/schema11.xml b/src/test/test-files/solr/conf/schema11.xml index ebc3bb98165..07fa6a2e87d 100755 --- a/src/test/test-files/solr/conf/schema11.xml +++ b/src/test/test-files/solr/conf/schema11.xml @@ -237,6 +237,16 @@ + + + + + + + + + + @@ -282,6 +292,12 @@ + + + + + +