SOLR-939: ValueSourceRangeFilter/Query, frange parser

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@757570 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2009-03-23 22:08:44 +00:00
parent a3e7952c84
commit 6bd4909102
21 changed files with 1007 additions and 38 deletions

View File

@ -189,6 +189,8 @@ New Features
32. SOLR-844: A SolrServer implementation to front-end multiple solr servers and provides load balancing and failover
support (Noble Paul, Mark Miller, hossman via shalin)
33. SOLR-939: ValueSourceRangeFilter/Query - filter based on values in a FieldCache entry or on any arbitrary function of field values. (yonik)
Optimizations
----------------------

View File

@ -22,8 +22,9 @@ import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.search.SortField;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.OrdFieldSource;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.search.function.*;
import org.apache.solr.search.QParser;
import org.apache.solr.util.DateMathParser;
import java.util.Map;
@ -330,5 +331,70 @@ public class DateField extends FieldType {
return (DateFormat) proto.clone();
}
}
@Override
public ValueSource getValueSource(SchemaField field, QParser parser) {
return new DateFieldSource(field.getName(), field.getType());
}
}
class DateFieldSource extends FieldCacheSource {
// NOTE: this is bad for serialization... but we currently need the fieldType for toInternal()
FieldType ft;
public DateFieldSource(String name, FieldType ft) {
super(name);
this.ft = ft;
}
public String description() {
return "date(" + field + ')';
}
public DocValues getValues(IndexReader reader) throws IOException {
return new StringIndexDocValues(this, reader, field) {
protected String toTerm(String readableValue) {
// needed for frange queries to work properly
return ft.toInternal(readableValue);
}
public float floatVal(int doc) {
return (float)intVal(doc);
}
public int intVal(int doc) {
int ord=order[doc];
return ord;
}
public long longVal(int doc) {
return (long)intVal(doc);
}
public double doubleVal(int doc) {
return (double)intVal(doc);
}
public String strVal(int doc) {
int ord=order[doc];
return ft.indexedToReadable(lookup[ord]);
}
public String toString(int doc) {
return description() + '=' + intVal(doc);
}
};
}
public boolean equals(Object o) {
return o instanceof DateFieldSource
&& super.equals(o);
}
private static int hcode = DateFieldSource.class.hashCode();
public int hashCode() {
return hcode + super.hashCode();
};
}

View File

@ -22,6 +22,7 @@ import org.apache.lucene.search.FieldCache;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.FieldCacheSource;
import org.apache.solr.search.function.DocValues;
import org.apache.solr.search.function.StringIndexDocValues;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.util.NumberUtils;
@ -93,12 +94,13 @@ class SortableDoubleFieldSource extends FieldCacheSource {
}
public DocValues getValues(IndexReader reader) throws IOException {
final FieldCache.StringIndex index = cache.getStringIndex(reader, field);
final int[] order = index.order;
final String[] lookup = index.lookup;
final double def = defVal;
return new DocValues() {
return new StringIndexDocValues(this, reader, field) {
protected String toTerm(String readableValue) {
return NumberUtils.double2sortableStr(readableValue);
}
public float floatVal(int doc) {
return (float)doubleVal(doc);
}

View File

@ -22,6 +22,7 @@ import org.apache.lucene.search.FieldCache;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.FieldCacheSource;
import org.apache.solr.search.function.DocValues;
import org.apache.solr.search.function.StringIndexDocValues;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.util.NumberUtils;
@ -93,12 +94,13 @@ class SortableFloatFieldSource extends FieldCacheSource {
}
public DocValues getValues(IndexReader reader) throws IOException {
final FieldCache.StringIndex index = cache.getStringIndex(reader, field);
final int[] order = index.order;
final String[] lookup = index.lookup;
final float def = defVal;
return new DocValues() {
return new StringIndexDocValues(this, reader, field) {
protected String toTerm(String readableValue) {
return NumberUtils.float2sortableStr(readableValue);
}
public float floatVal(int doc) {
int ord=order[doc];
return ord==0 ? def : NumberUtils.SortableStr2float(lookup[ord]);

View File

@ -22,6 +22,7 @@ import org.apache.lucene.search.FieldCache;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.FieldCacheSource;
import org.apache.solr.search.function.DocValues;
import org.apache.solr.search.function.StringIndexDocValues;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.util.NumberUtils;
@ -97,12 +98,13 @@ class SortableIntFieldSource extends FieldCacheSource {
}
public DocValues getValues(IndexReader reader) throws IOException {
final FieldCache.StringIndex index = cache.getStringIndex(reader, field);
final int[] order = index.order;
final String[] lookup = index.lookup;
final int def = defVal;
return new DocValues() {
return new StringIndexDocValues(this, reader, field) {
protected String toTerm(String readableValue) {
return NumberUtils.int2sortableStr(readableValue);
}
public float floatVal(int doc) {
return (float)intVal(doc);
}

View File

@ -22,6 +22,7 @@ import org.apache.lucene.search.FieldCache;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.FieldCacheSource;
import org.apache.solr.search.function.DocValues;
import org.apache.solr.search.function.StringIndexDocValues;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.util.NumberUtils;
@ -94,12 +95,13 @@ class SortableLongFieldSource extends FieldCacheSource {
}
public DocValues getValues(IndexReader reader) throws IOException {
final FieldCache.StringIndex index = cache.getStringIndex(reader, field);
final int[] order = index.order;
final String[] lookup = index.lookup;
final long def = defVal;
return new DocValues() {
return new StringIndexDocValues(this, reader, field) {
protected String toTerm(String readableValue) {
return NumberUtils.long2sortableStr(readableValue);
}
public float floatVal(int doc) {
return (float)longVal(doc);
}

View File

@ -19,8 +19,15 @@ package org.apache.solr.schema;
import org.apache.lucene.search.SortField;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.FieldCacheSource;
import org.apache.solr.search.function.DocValues;
import org.apache.solr.search.function.StringIndexDocValues;
import org.apache.solr.search.QParser;
import org.apache.solr.util.NumberUtils;
import java.util.Map;
import java.io.IOException;
@ -43,4 +50,64 @@ public class StrField extends CompressableField {
public void write(TextResponseWriter writer, String name, Fieldable f) throws IOException {
writer.writeStr(name, f.stringValue(), true);
}
public ValueSource getValueSource(SchemaField field, QParser parser) {
return super.getValueSource(field, parser);
}
}
class StrFieldSource extends FieldCacheSource {
public StrFieldSource(String field) {
super(field);
}
public String description() {
return "str(" + field + ')';
}
public DocValues getValues(IndexReader reader) throws IOException {
return new StringIndexDocValues(this, reader, field) {
protected String toTerm(String readableValue) {
return readableValue;
}
public float floatVal(int doc) {
return (float)intVal(doc);
}
public int intVal(int doc) {
int ord=order[doc];
return ord;
}
public long longVal(int doc) {
return (long)intVal(doc);
}
public double doubleVal(int doc) {
return (double)intVal(doc);
}
public String strVal(int doc) {
int ord=order[doc];
return lookup[ord];
}
public String toString(int doc) {
return description() + '=' + strVal(doc);
}
};
}
public boolean equals(Object o) {
return o instanceof StrFieldSource
&& super.equals(o);
}
private static int hcode = SortableFloatFieldSource.class.hashCode();
public int hashCode() {
return hcode + super.hashCode();
};
}

View File

@ -65,7 +65,7 @@ public class BoostQParserPlugin extends QParserPlugin {
public String[] getDefaultHighlightFields() {
return baseParser.getDefaultHighlightFields();
}
public Query getHighlightQuery() throws ParseException {
return baseParser.getHighlightQuery();
}

View File

@ -0,0 +1,69 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.function.*;
/**
* Create a range query over a function.
* <br>Other parameters:
* <br><code>l</code>, the lower bound, optional)
* <br><code>u</code>, the upper bound, optional)
* <br><code>incl</code>, include the lower bound: true/false, optional, default=true
* <br><code>incl</code>, include the upper bound: true/false, optional, default=true
* <br>Example: <code>{!frange l=1000 u=50000}myfield</code>
*/
public class FunctionRangeQParserPlugin extends QParserPlugin {
public static String NAME = "frange";
public void init(NamedList args) {
}
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
return new QParser(qstr, localParams, params, req) {
ValueSource vs;
String funcStr;
public Query parse() throws ParseException {
funcStr = localParams.get(QueryParsing.V, null);
Query funcQ = subQuery(funcStr, FunctionQParserPlugin.NAME).parse();
if (funcQ instanceof FunctionQuery) {
vs = ((FunctionQuery)funcQ).getValueSource();
} else {
vs = new QueryValueSource(funcQ, 0.0f);
}
String l = localParams.get("l");
String u = localParams.get("u");
boolean includeLower = localParams.getBool("incl",true);
boolean includeUpper = localParams.getBool("incu",true);
// TODO: add a score=val option to allow score to be the value
ValueSourceRangeFilter rf = new ValueSourceRangeFilter(vs, l, u, includeLower, includeUpper);
ConstantScoreQuery csq = new ConstantScoreQuery(rf);
return csq;
}
};
}
}

View File

@ -35,6 +35,7 @@ public abstract class QParserPlugin implements NamedListInitializedPlugin {
FieldQParserPlugin.NAME, FieldQParserPlugin.class,
RawQParserPlugin.NAME, RawQParserPlugin.class,
NestedQParserPlugin.NAME, NestedQParserPlugin.class,
FunctionRangeQParserPlugin.NAME, FunctionRangeQParserPlugin.class,
};
/** return a {@link QParser} */

View File

@ -17,7 +17,11 @@
package org.apache.solr.search.function;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.*;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.util.NumberUtils;
import java.io.IOException;
/**
* Represents field values as different types.
@ -33,6 +37,7 @@ import org.apache.lucene.search.Explanation;
// - For caching, Query objects are often used as keys... you don't
// want the Query carrying around big objects
public abstract class DocValues {
public byte byteVal(int doc) { throw new UnsupportedOperationException(); }
public short shortVal(int doc) { throw new UnsupportedOperationException(); }
@ -42,7 +47,76 @@ public abstract class DocValues {
public double doubleVal(int doc) { throw new UnsupportedOperationException(); }
public String strVal(int doc) { throw new UnsupportedOperationException(); }
public abstract String toString(int doc);
public Explanation explain(int doc) {
return new Explanation(floatVal(doc), toString(doc));
}
public ValueSourceScorer getScorer(IndexReader reader) {
return new ValueSourceScorer(reader, this);
}
// A RangeValueSource can't easily be a ValueSource that takes another ValueSource
// because it needs different behavior depending on the type of fields. There is also
// a setup cost - parsing and normalizing params, and doing a binary search on the StringIndex.
public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
float lower;
float upper;
if (lowerVal == null) {
lower = Float.NEGATIVE_INFINITY;
} else {
lower = Float.parseFloat(lowerVal);
}
if (upperVal == null) {
upper = Float.POSITIVE_INFINITY;
} else {
upper = Float.parseFloat(upperVal);
}
final float l = lower;
final float u = upper;
if (includeLower && includeUpper) {
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
float docVal = floatVal(doc);
return docVal >= l && docVal <= u;
}
};
}
else if (includeLower && !includeUpper) {
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
float docVal = floatVal(doc);
return docVal >= l && docVal < u;
}
};
}
else if (!includeLower && includeUpper) {
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
float docVal = floatVal(doc);
return docVal > l && docVal <= u;
}
};
}
else {
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
float docVal = floatVal(doc);
return docVal > l && docVal < u;
}
};
}
}
}

View File

@ -74,7 +74,68 @@ public class DoubleFieldSource extends FieldCacheSource {
public String toString(int doc) {
return description() + '=' + floatVal(doc);
}
};
@Override
public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
double lower,upper;
if (lowerVal==null) {
lower = Double.NEGATIVE_INFINITY;
} else {
lower = Double.parseDouble(lowerVal);
}
if (upperVal==null) {
upper = Double.POSITIVE_INFINITY;
} else {
upper = Double.parseDouble(upperVal);
}
final double l = lower;
final double u = upper;
if (includeLower && includeUpper) {
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
double docVal = doubleVal(doc);
return docVal >= l && docVal <= u;
}
};
}
else if (includeLower && !includeUpper) {
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
double docVal = doubleVal(doc);
return docVal >= l && docVal < u;
}
};
}
else if (!includeLower && includeUpper) {
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
double docVal = doubleVal(doc);
return docVal > l && docVal <= u;
}
};
}
else {
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
double docVal = doubleVal(doc);
return docVal > l && docVal < u;
}
};
}
}
};
}
public boolean equals(Object o) {
@ -86,11 +147,9 @@ public class DoubleFieldSource extends FieldCacheSource {
}
public int hashCode() {
int h = parser == null ? Float.class.hashCode() : parser.getClass().hashCode();
int h = parser == null ? Double.class.hashCode() : parser.getClass().hashCode();
h += super.hashCode();
return h;
}
;
}

View File

@ -46,6 +46,7 @@ public class IntFieldSource extends FieldCacheSource {
return "int(" + field + ')';
}
public DocValues getValues(IndexReader reader) throws IOException {
final int[] arr = (parser==null) ?
cache.getInts(reader, field) :
@ -75,6 +76,39 @@ public class IntFieldSource extends FieldCacheSource {
return description() + '=' + intVal(doc);
}
@Override
public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
int lower,upper;
// instead of using separate comparison functions, adjust the endpoints.
if (lowerVal==null) {
lower = Integer.MIN_VALUE;
} else {
lower = Integer.parseInt(lowerVal);
if (!includeLower && lower < Integer.MAX_VALUE) lower++;
}
if (upperVal==null) {
upper = Integer.MAX_VALUE;
} else {
upper = Integer.parseInt(upperVal);
if (!includeUpper && upper > Integer.MIN_VALUE) upper--;
}
final int ll = lower;
final int uu = upper;
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
int val = arr[doc];
// only check for deleted if it's the default value
// if (val==0 && reader.isDeleted(doc)) return false;
return val >= ll && val <= uu;
}
};
}
};
}

View File

@ -75,6 +75,43 @@ public class LongFieldSource extends FieldCacheSource {
public String toString(int doc) {
return description() + '=' + floatVal(doc);
}
@Override
public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
long lower,upper;
// instead of using separate comparison functions, adjust the endpoints.
if (lowerVal==null) {
lower = Long.MIN_VALUE;
} else {
lower = Long.parseLong(lowerVal);
if (!includeLower && lower < Long.MAX_VALUE) lower++;
}
if (upperVal==null) {
upper = Long.MAX_VALUE;
} else {
upper = Long.parseLong(upperVal);
if (!includeUpper && upper > Long.MIN_VALUE) upper--;
}
final long ll = lower;
final long uu = upper;
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
long val = arr[doc];
// only check for deleted if it's the default value
// if (val==0 && reader.isDeleted(doc)) return false;
return val >= ll && val <= uu;
}
};
}
};
}
@ -87,11 +124,9 @@ public class LongFieldSource extends FieldCacheSource {
}
public int hashCode() {
int h = parser == null ? Float.class.hashCode() : parser.getClass().hashCode();
int h = parser == null ? Long.class.hashCode() : parser.getClass().hashCode();
h += super.hashCode();
return h;
}
;
}

View File

@ -50,28 +50,32 @@ public class OrdFieldSource extends ValueSource {
return "ord(" + field + ')';
}
public DocValues getValues(IndexReader reader) throws IOException {
final int[] arr = FieldCache.DEFAULT.getStringIndex(reader, field).order;
return new DocValues() {
return new StringIndexDocValues(this, reader, field) {
protected String toTerm(String readableValue) {
return readableValue;
}
public float floatVal(int doc) {
return (float)arr[doc];
return (float)order[doc];
}
public int intVal(int doc) {
return (int)arr[doc];
return order[doc];
}
public long longVal(int doc) {
return (long)arr[doc];
return (long)order[doc];
}
public double doubleVal(int doc) {
return (double)arr[doc];
return (double)order[doc];
}
public String strVal(int doc) {
// the string value of the ordinal, not the string itself
return Integer.toString(arr[doc]);
return Integer.toString(order[doc]);
}
public String toString(int doc) {
@ -81,9 +85,7 @@ public class OrdFieldSource extends ValueSource {
}
public boolean equals(Object o) {
if (o.getClass() != OrdFieldSource.class) return false;
OrdFieldSource other = (OrdFieldSource)o;
return this.field.equals(field);
return o.getClass() == OrdFieldSource.class && this.field.equals(field);
}
private static final int hcode = OrdFieldSource.class.hashCode();

View File

@ -0,0 +1,86 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search.function;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.ExtendedFieldCache;
import org.apache.lucene.index.IndexReader;
import java.io.IOException;
/** Internal class, subject to change.
* Serves as base class for DocValues based on StringIndex
**/
public abstract class StringIndexDocValues extends DocValues {
protected final FieldCache.StringIndex index;
protected final int[] order;
protected final String[] lookup;
protected final ValueSource vs;
public StringIndexDocValues(ValueSource vs, IndexReader reader, String field) throws IOException {
index = ExtendedFieldCache.EXT_DEFAULT.getStringIndex(reader, field);
order = index.order;
lookup = index.lookup;
this.vs = vs;
}
protected abstract String toTerm(String readableValue);
@Override
public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
// TODO: are lowerVal and upperVal in indexed form or not?
lowerVal = lowerVal == null ? null : toTerm(lowerVal);
upperVal = upperVal == null ? null : toTerm(upperVal);
int lower = Integer.MIN_VALUE;
if (lowerVal != null) {
lower = index.binarySearchLookup(lowerVal);
if (lower < 0) {
lower = -lower-1;
} else if (!includeLower) {
lower++;
}
}
int upper = Integer.MAX_VALUE;
if (upperVal != null) {
upper = index.binarySearchLookup(upperVal);
if (upper < 0) {
upper = -upper-2;
} else if (!includeUpper) {
upper--;
}
}
final int ll = lower;
final int uu = upper;
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
int ord = order[doc];
return ord >= ll && ord <= uu;
}
};
}
public String toString(int doc) {
return vs.description() + '=' + strVal(doc);
}
}

View File

@ -18,6 +18,7 @@
package org.apache.solr.search.function;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.*;
import org.apache.solr.search.function.DocValues;
import java.io.IOException;
@ -46,3 +47,62 @@ public abstract class ValueSource implements Serializable {
}
}
class ValueSourceScorer extends Scorer {
protected IndexReader reader;
private int doc = -1;
protected final int maxDoc;
protected final DocValues values;
protected boolean checkDeletes;
protected ValueSourceScorer(IndexReader reader, DocValues values) {
super(null);
this.reader = reader;
this.maxDoc = reader.maxDoc();
this.values = values;
setCheckDeletes(true);
}
public IndexReader getReader() { return reader; }
public void setCheckDeletes(boolean checkDeletes) {
this.checkDeletes = checkDeletes && reader.hasDeletions();
}
public boolean matches(int doc) {
return (!checkDeletes || !reader.isDeleted(maxDoc)) && matchesValue(doc);
}
public boolean matchesValue(int doc) {
return true;
}
public int doc() {
return doc;
}
public boolean next() {
for(;;) {
doc++;
if (doc >= maxDoc) return false;
if (matches(doc)) return true;
}
}
public boolean skipTo(int target) {
doc = target-1;
return next();
}
public float score() throws IOException {
return values.floatVal(doc);
}
public Explanation explain(int doc) throws IOException {
return values.explain(doc);
}
}

View File

@ -0,0 +1,94 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search.function;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.index.IndexReader;
import java.io.IOException;
/**
* RangeFilter over a ValueSource.
*/
public class ValueSourceRangeFilter extends Filter {
private final ValueSource valueSource;
private final String lowerVal;
private final String upperVal;
private final boolean includeLower;
private final boolean includeUpper;
public ValueSourceRangeFilter(ValueSource valueSource,
String lowerVal,
String upperVal,
boolean includeLower,
boolean includeUpper) {
this.valueSource = valueSource;
this.lowerVal = lowerVal;
this.upperVal = upperVal;
this.includeLower = lowerVal != null && includeLower;
this.includeUpper = upperVal != null && includeUpper;
}
public DocIdSet getDocIdSet(final IndexReader reader) throws IOException {
return new DocIdSet() {
public DocIdSetIterator iterator() throws IOException {
return valueSource.getValues(reader).getRangeScorer(reader, lowerVal, upperVal, includeLower, includeUpper);
}
};
}
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("frange(");
sb.append(valueSource);
sb.append("):");
sb.append(includeLower ? '[' : '{');
sb.append(lowerVal == null ? "*" : lowerVal);
sb.append(" TO ");
sb.append(upperVal == null ? "*" : upperVal);
sb.append(includeUpper ? ']' : '}');
return sb.toString();
}
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof ValueSourceRangeFilter)) return false;
ValueSourceRangeFilter other = (ValueSourceRangeFilter)o;
if (!this.valueSource.equals(other.valueSource)
|| this.includeLower != other.includeLower
|| this.includeUpper != other.includeUpper
) { return false; }
if (this.lowerVal != null ? !this.lowerVal.equals(other.lowerVal) : other.lowerVal != null) return false;
if (this.upperVal != null ? !this.upperVal.equals(other.upperVal) : other.upperVal != null) return false;
return true;
}
public int hashCode() {
int h = valueSource.hashCode();
h += lowerVal != null ? lowerVal.hashCode() : 0x572353db;
h = (h << 16) | (h >>> 16); // rotate to distinguish lower from upper
h += (upperVal != null ? (upperVal.hashCode()) : 0xe16fe9e7);
h += (includeLower ? 0xdaa47978 : 0)
+ (includeUpper ? 0x9e634b57 : 0);
return h;
}
}

View File

@ -21,6 +21,8 @@ package org.apache.solr.util;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.util.XML;
import org.apache.solr.request.*;
import org.apache.solr.util.TestHarness;
@ -30,6 +32,8 @@ import junit.framework.TestCase;
import javax.xml.xpath.XPathExpressionException;
import java.io.*;
import java.util.List;
import java.util.ArrayList;
/**
* An Abstract base class that makes writing Solr JUnit tests "easier"
@ -222,6 +226,21 @@ public abstract class AbstractSolrTestCase extends TestCase {
Doc d = doc(fieldsAndValues);
return add(d);
}
/**
* Generates a simple &lt;add&gt;&lt;doc&gt;... XML String with no options
*/
public String adoc(SolrInputDocument sdoc) {
List<String> fields = new ArrayList<String>();
for (SolrInputField sf : sdoc) {
for (Object o : sf.getValues()) {
fields.add(sf.getName());
fields.add(o.toString());
}
}
return adoc(fields.toArray(new String[fields.size()]));
}
/**
* Generates an &lt;add&gt;&lt;doc&gt;... XML String with options

View File

@ -0,0 +1,277 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import org.apache.solr.util.AbstractSolrTestCase;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.request.SolrQueryResponse;
import java.util.*;
import junit.framework.TestCase;
public class TestRangeQuery extends AbstractSolrTestCase {
public String getSchemaFile() { return "schema11.xml"; }
public String getSolrConfigFile() { return "solrconfig.xml"; }
public String getCoreName() { return "basic"; }
public void setUp() throws Exception {
// if you override setUp or tearDown, you better call
// the super classes version
super.setUp();
}
public void tearDown() throws Exception {
// if you override setUp or tearDown, you better call
// the super classes version
super.tearDown();
}
Random r = new Random(1);
void addInt(SolrInputDocument doc, int l, int u, String... fields) {
int v=0;
if (0==l && l==u) {
v=r.nextInt();
} else {
v=r.nextInt(u-l)+l;
}
for (String field : fields) {
doc.addField(field, v);
}
}
interface DocProcessor {
public void process(SolrInputDocument doc);
}
public void createIndex(int nDocs, DocProcessor proc) {
for (int i=0; i<nDocs; i++) {
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", ""+i);
proc.process(doc);
assertU(adoc(doc));
}
}
public void testRangeQueries() throws Exception {
// ensure that we aren't losing precision on any fields in addition to testing other non-numeric fields
// that aren't tested in testRandomRangeQueries()
int i=2000000000;
long l=500000000000000000L;
double d=0.3333333333333333;
// first 3 values will be indexed, the last two won't be
String[] ints = {""+(i-1), ""+(i), ""+(i+1), ""+(i-2), ""+(i+2)};
String[] longs = {""+(l-1), ""+(l), ""+(l+1), ""+(l-2), ""+(l+2)};
String[] doubles = {""+(d-1e-16), ""+(d), ""+(d+1e-16), ""+(d-2e-16), ""+(d+2e-16)};
String[] strings = {"aaa","bbb","ccc", "aa","cccc" };
String[] dates = {"1999-12-31T23:59:59.999Z","2000-01-01T00:00:00.000Z","2000-01-01T00:00:00.001Z", "1999-12-31T23:59:59.998Z","2000-01-01T00:00:00.002Z" };
// fields that normal range queries should work on
Map<String,String[]> norm_fields = new HashMap<String,String[]>();
norm_fields.put("foo_i", ints);
norm_fields.put("foo_l", longs);
norm_fields.put("foo_d", doubles);
norm_fields.put("foo_ti", ints);
norm_fields.put("foo_tl", longs);
norm_fields.put("foo_td", doubles);
norm_fields.put("foo_s", strings);
norm_fields.put("foo_dt", dates);
// fields that frange queries should work on
Map<String,String[]> frange_fields = new HashMap<String,String[]>();
frange_fields.put("foo_i", ints);
frange_fields.put("foo_l", longs);
frange_fields.put("foo_d", doubles);
frange_fields.put("foo_pi", ints);
frange_fields.put("foo_pl", longs);
frange_fields.put("foo_pd", doubles);
frange_fields.put("foo_s", strings);
frange_fields.put("foo_dt", dates);
Map<String,String[]> all_fields = new HashMap<String,String[]>();
all_fields.putAll(norm_fields);
all_fields.putAll(frange_fields);
for (int j=0; j<ints.length-2; j++) {
List<String> fields = new ArrayList<String>();
fields.add("id");
fields.add(""+j);
for (Map.Entry<String,String[]> entry : all_fields.entrySet()) {
fields.add(entry.getKey());
fields.add(entry.getValue()[j]);
}
assertU(adoc(fields.toArray(new String[fields.size()])));
}
assertU(commit());
// simple test of a function rather than just the field
assertQ(req("{!frange l=0 u=2}id"), "*[count(//doc)=3]");
assertQ(req("{!frange l=0 u=2}product(id,2)"), "*[count(//doc)=2]");
assertQ(req("{!frange l=100 u=102}sum(id,100)"), "*[count(//doc)=3]");
for (Map.Entry<String,String[]> entry : norm_fields.entrySet()) {
String f = entry.getKey();
String[] v = entry.getValue();
assertQ(req(f + ":[* TO *]" ), "*[count(//doc)=3]");
assertQ(req(f + ":["+v[0]+" TO "+v[2]+"]"), "*[count(//doc)=3]");
assertQ(req(f + ":["+v[1]+" TO "+v[2]+"]"), "*[count(//doc)=2]");
assertQ(req(f + ":["+v[0]+" TO "+v[1]+"]"), "*[count(//doc)=2]");
assertQ(req(f + ":["+v[0]+" TO "+v[0]+"]"), "*[count(//doc)=1]");
assertQ(req(f + ":["+v[1]+" TO "+v[1]+"]"), "*[count(//doc)=1]");
assertQ(req(f + ":["+v[2]+" TO "+v[2]+"]"), "*[count(//doc)=1]");
assertQ(req(f + ":["+v[3]+" TO "+v[3]+"]"), "*[count(//doc)=0]");
assertQ(req(f + ":["+v[4]+" TO "+v[4]+"]"), "*[count(//doc)=0]");
assertQ(req(f + ":{"+v[0]+" TO "+v[2]+"}"), "*[count(//doc)=1]");
assertQ(req(f + ":{"+v[1]+" TO "+v[2]+"}"), "*[count(//doc)=0]");
assertQ(req(f + ":{"+v[0]+" TO "+v[1]+"}"), "*[count(//doc)=0]");
assertQ(req(f + ":{"+v[3]+" TO "+v[4]+"}"), "*[count(//doc)=3]");
}
for (Map.Entry<String,String[]> entry : frange_fields.entrySet()) {
String f = entry.getKey();
String[] v = entry.getValue();
assertQ(req("{!frange}"+f ), "*[count(//doc)=3]");
assertQ(req("{!frange" + " l="+v[0]+"}"+f ), "*[count(//doc)=3]");
assertQ(req("{!frange" + " l="+v[1]+"}"+f ), "*[count(//doc)=2]");
assertQ(req("{!frange" + " l="+v[2]+"}"+f ), "*[count(//doc)=1]");
assertQ(req("{!frange" + " l="+v[3]+"}"+f ), "*[count(//doc)=3]");
assertQ(req("{!frange" + " l="+v[4]+"}"+f ), "*[count(//doc)=0]");
assertQ(req("{!frange" + " u="+v[0]+"}"+f ), "*[count(//doc)=1]");
assertQ(req("{!frange" + " u="+v[1]+"}"+f ), "*[count(//doc)=2]");
assertQ(req("{!frange" + " u="+v[2]+"}"+f ), "*[count(//doc)=3]");
assertQ(req("{!frange" + " u="+v[3]+"}"+f ), "*[count(//doc)=0]");
assertQ(req("{!frange" + " u="+v[4]+"}"+f ), "*[count(//doc)=3]");
assertQ(req("{!frange incl=false" + " l="+v[0]+"}"+f ), "*[count(//doc)=2]");
assertQ(req("{!frange incl=false" + " l="+v[1]+"}"+f ), "*[count(//doc)=1]");
assertQ(req("{!frange incl=false" + " l="+v[2]+"}"+f ), "*[count(//doc)=0]");
assertQ(req("{!frange incl=false" + " l="+v[3]+"}"+f ), "*[count(//doc)=3]");
assertQ(req("{!frange incl=false" + " l="+v[4]+"}"+f ), "*[count(//doc)=0]");
assertQ(req("{!frange incu=false" + " u="+v[0]+"}"+f ), "*[count(//doc)=0]");
assertQ(req("{!frange incu=false" + " u="+v[1]+"}"+f ), "*[count(//doc)=1]");
assertQ(req("{!frange incu=false" + " u="+v[2]+"}"+f ), "*[count(//doc)=2]");
assertQ(req("{!frange incu=false" + " u="+v[3]+"}"+f ), "*[count(//doc)=0]");
assertQ(req("{!frange incu=false" + " u="+v[4]+"}"+f ), "*[count(//doc)=3]");
assertQ(req("{!frange incl=true incu=true" + " l=" +v[0] +" u="+v[2]+"}"+f ), "*[count(//doc)=3]");
assertQ(req("{!frange incl=false incu=false" + " l=" +v[0] +" u="+v[2]+"}"+f ), "*[count(//doc)=1]");
assertQ(req("{!frange incl=false incu=false" + " l=" +v[3] +" u="+v[4]+"}"+f ), "*[count(//doc)=3]");
}
}
public void testRandomRangeQueries() throws Exception {
String handler="";
final String[] fields = {"foo_s","foo_i","foo_l","foo_f","foo_d" // SortableIntField, etc
,"foo_pi","foo_pl","foo_pf","foo_pd" // plain int IntField, etc
,"foo_ti","foo_tl","foo_tf","foo_td" // trie numer fields
};
final int l=5;
final int u=25;
createIndex(15, new DocProcessor() {
public void process(SolrInputDocument doc) {
addInt(doc, l,u, fields);
}
});
assertU(commit());
// fields that a normal range query will work correctly on
String[] norm_fields = {
"foo_i","foo_l","foo_f","foo_d"
,"foo_ti","foo_tl","foo_tf","foo_td"
};
// fields that a value source range query should work on
String[] frange_fields = {"foo_i","foo_l","foo_f","foo_d",
"foo_pi","foo_pl","foo_pf","foo_pd"};
for (int i=0; i<1000; i++) {
int lower = l + r.nextInt(u-l+10)-5;
int upper = lower + r.nextInt(u+5-lower);
boolean lowerMissing = r.nextInt(10)==1;
boolean upperMissing = r.nextInt(10)==1;
boolean inclusive = lowerMissing || upperMissing || r.nextBoolean();
// lower=2; upper=2; inclusive=true;
// inclusive=true; lowerMissing=true; upperMissing=true;
List<String> qs = new ArrayList<String>();
for (String field : norm_fields) {
String q = field + ':' + (inclusive?'[':'{')
+ (lowerMissing?"*":lower)
+ " TO "
+ (upperMissing?"*":upper)
+ (inclusive?']':'}');
qs.add(q);
}
for (String field : frange_fields) {
String q = "{!frange v="+field
+ (lowerMissing?"":(" l="+lower))
+ (upperMissing?"":(" u="+upper))
+ (inclusive?"":" incl=false")
+ (inclusive?"":" incu=false")
+ "}";
qs.add(q);
}
SolrQueryResponse last=null;
for (String q : qs) {
// System.out.println("QUERY="+q);
SolrQueryResponse qr = h.queryAndResponse(handler, req("q",q,"rows","1000"));
if (last != null) {
// we only test if the same docs matched since some queries will include factors like idf, etc.
sameDocs((DocSet)qr.getValues().get("response"), (DocSet)last.getValues().get("response"));
}
last = qr;
}
}
}
static boolean sameDocs(DocSet a, DocSet b) {
DocIterator i = a.iterator();
// System.out.println("SIZES="+a.size() + "," + b.size());
assertEquals(a.size(), b.size());
while (i.hasNext()) {
int doc = i.nextDoc();
if (!b.exists(doc)) {
TestCase.fail("Missing doc " + doc);
}
// System.out.println("MATCH! " + doc);
}
return true;
}
}

View File

@ -237,6 +237,16 @@
<fieldType name="file" keyField="id" defVal="1" stored="false" indexed="false" class="solr.ExternalFileField" valType="float"/>
<fieldType name="tint" class="solr.TrieField" type="integer" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<fieldType name="tfloat" class="solr.TrieField" type="float" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<fieldType name="tlong" class="solr.TrieField" type="long" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<fieldType name="tdouble" class="solr.TrieField" type="double" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<fieldType name="tdouble4" class="solr.TrieField" type="double" precisionStep="4" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<fieldType name="tdate" class="solr.TrieField" type="date" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
</types>
@ -282,6 +292,12 @@
<dynamicField name="*_pf" type="float" indexed="true" stored="true"/>
<dynamicField name="*_pd" type="double" indexed="true" stored="true"/>
<dynamicField name="*_ti" type="tint" indexed="true" stored="true"/>
<dynamicField name="*_tl" type="tlong" indexed="true" stored="true"/>
<dynamicField name="*_tf" type="tfloat" indexed="true" stored="true"/>
<dynamicField name="*_td" type="tdouble" indexed="true" stored="true"/>
<dynamicField name="*_tdt" type="tdate" indexed="true" stored="true"/>
<dynamicField name="*_t" type="text" indexed="true" stored="true"/>
<dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
<dynamicField name="*_dt" type="date" indexed="true" stored="true"/>