Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5247: expressions should use DoubleDocValues not FunctionValues directly 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1528167 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2013-10-01 18:14:00 +00:00
parent d36147eca7
commit 9db7442d64
12 changed files with 371 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionComparator.java
View File

@ -49,7 +49,7 @@ class ExpressionComparator extends FieldComparator<Double> {
try {
Map<String,Object> context = new HashMap<String,Object>();
assert scorer != null;
context.put("scorer", new ScoreFunctionValues(scorer));
context.put("scorer", scorer);
scores = source.getValues(context, readerContext);
} catch (IOException e) {
throw new RuntimeException(e);

17
lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionFunctionValues.java
View File

@ -17,16 +17,19 @@ package org.apache.lucene.expressions;
*/
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
/** A {@link FunctionValues} which evaluates an expression */
class ExpressionFunctionValues extends FunctionValues {
class ExpressionFunctionValues extends DoubleDocValues {
final Expression expression;
final FunctionValues[] functionValues;
int currentDocument = -1;
double currentValue;
public ExpressionFunctionValues(Expression expression, FunctionValues[] functionValues) {
ExpressionFunctionValues(ValueSource parent, Expression expression, FunctionValues[] functionValues) {
super(parent);
if (expression == null) {
throw new NullPointerException();
}
@ -46,14 +49,4 @@ class ExpressionFunctionValues extends FunctionValues {
return currentValue;
}
@Override
public Object objectVal(int doc) {
return doubleVal(doc);
}
@Override
public String toString(int document) {
return "ExpressionFunctionValues(" + document + ": " + objectVal(document) + ")";
}
}

48
lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionValueSource.java
View File

@ -17,6 +17,7 @@ package org.apache.lucene.expressions;
*/
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
@ -79,7 +80,7 @@ final class ExpressionValueSource extends ValueSource {
externalValues[i] = values;
}
return new ExpressionFunctionValues(expression, externalValues);
return new ExpressionFunctionValues(this, expression, externalValues);
}
@Override
@ -89,19 +90,48 @@ final class ExpressionValueSource extends ValueSource {
@Override
public String description() {
return "ExpressionValueSource(" + expression.sourceText + ")";
return "expr(" + expression.sourceText + ")";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result
+ ((expression == null) ? 0 : expression.hashCode());
result = prime * result + (needsScores ? 1231 : 1237);
result = prime * result + Arrays.hashCode(variables);
return result;
}
@Override
public int hashCode() {
return System.identityHashCode(this);
}
@Override
public boolean equals(Object obj) {
return obj == this;
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
ExpressionValueSource other = (ExpressionValueSource) obj;
if (expression == null) {
if (other.expression != null) {
return false;
}
} else if (!expression.equals(other.expression)) {
return false;
}
if (needsScores != other.needsScores) {
return false;
}
if (!Arrays.equals(variables, other.variables)) {
return false;
}
return true;
}
boolean needsScores() {
return needsScores;
}

12
lucene/expressions/src/java/org/apache/lucene/expressions/ScoreFunctionValues.java
View File

@ -19,15 +19,18 @@ package org.apache.lucene.expressions;
import java.io.IOException;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
import org.apache.lucene.search.Scorer;
/**
* A utility class to allow expressions to access the score as a {@link FunctionValues}.
*/
class ScoreFunctionValues extends FunctionValues {
class ScoreFunctionValues extends DoubleDocValues {
final Scorer scorer;
ScoreFunctionValues(Scorer scorer) {
ScoreFunctionValues(ValueSource parent, Scorer scorer) {
super(parent);
this.scorer = scorer;
}
@ -40,9 +43,4 @@ class ScoreFunctionValues extends FunctionValues {
throw new RuntimeException(exception);
}
}
@Override
public String toString(int document) {
return "ScoreFunctionValues(" + document + ": " + doubleVal(document) + ")";
}
}

11
lucene/expressions/src/java/org/apache/lucene/expressions/ScoreValueSource.java
View File

@ -20,27 +20,28 @@ package org.apache.lucene.expressions;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.Scorer;
import java.io.IOException;
import java.util.Map;
/**
* A {@link ValueSource} which uses the {@link ScoreFunctionValues} passed through
* A {@link ValueSource} which uses the {@link Scorer} passed through
* the context map by {@link ExpressionComparator}.
*/
@SuppressWarnings({"rawtypes"})
class ScoreValueSource extends ValueSource {
/**
* <code>context</code> must contain a key "scorer" which is a {@link FunctionValues}.
* <code>context</code> must contain a key "scorer" which is a {@link Scorer}.
*/
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
FunctionValues v = (FunctionValues) context.get("scorer");
Scorer v = (Scorer) context.get("scorer");
if (v == null) {
throw new IllegalStateException("Expressions referencing the score can only be used for sorting");
}
return v;
return new ScoreFunctionValues(this, v);
}
@Override
@ -55,6 +56,6 @@ class ScoreValueSource extends ValueSource {
@Override
public String description() {
return "ValueSource to expose scorer passed by ExpressionComparator";
return "score()";
}
}

177
lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java Normal file
View File

@ -0,0 +1,177 @@
package org.apache.lucene.expressions;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.HashMap;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.expressions.js.JavascriptCompiler;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.ValueSourceScorer;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.SortField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
@SuppressCodecs("Lucene3x")
public class TestExpressionValueSource extends LuceneTestCase {
DirectoryReader reader;
Directory dir;
@Override
public void setUp() throws Exception {
super.setUp();
dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
iwc.setMergePolicy(newLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
Document doc = new Document();
doc.add(newStringField("id", "1", Field.Store.YES));
doc.add(newTextField("body", "some contents and more contents", Field.Store.NO));
doc.add(new NumericDocValuesField("popularity", 5));
iw.addDocument(doc);
doc = new Document();
doc.add(newStringField("id", "2", Field.Store.YES));
doc.add(newTextField("body", "another document with different contents", Field.Store.NO));
doc.add(new NumericDocValuesField("popularity", 20));
iw.addDocument(doc);
doc = new Document();
doc.add(newStringField("id", "3", Field.Store.YES));
doc.add(newTextField("body", "crappy contents", Field.Store.NO));
doc.add(new NumericDocValuesField("popularity", 2));
iw.addDocument(doc);
iw.forceMerge(1);
reader = iw.getReader();
iw.close();
}
@Override
public void tearDown() throws Exception {
reader.close();
dir.close();
super.tearDown();
}
public void testTypes() throws Exception {
Expression expr = JavascriptCompiler.compile("2*popularity");
SimpleBindings bindings = new SimpleBindings();
bindings.add(new SortField("popularity", SortField.Type.LONG));
ValueSource vs = expr.getValueSource(bindings);
assertEquals(1, reader.leaves().size());
AtomicReaderContext leaf = reader.leaves().get(0);
FunctionValues values = vs.getValues(new HashMap<String,Object>(), leaf);
assertEquals(10, values.doubleVal(0), 0);
assertEquals(10, values.floatVal(0), 0);
assertEquals(10, values.longVal(0));
assertEquals(10, values.intVal(0));
assertEquals(10, values.shortVal(0));
assertEquals(10, values.byteVal(0));
assertEquals("10.0", values.strVal(0));
assertEquals(new Double(10), values.objectVal(0));
assertEquals(40, values.doubleVal(1), 0);
assertEquals(40, values.floatVal(1), 0);
assertEquals(40, values.longVal(1));
assertEquals(40, values.intVal(1));
assertEquals(40, values.shortVal(1));
assertEquals(40, values.byteVal(1));
assertEquals("40.0", values.strVal(1));
assertEquals(new Double(40), values.objectVal(1));
assertEquals(4, values.doubleVal(2), 0);
assertEquals(4, values.floatVal(2), 0);
assertEquals(4, values.longVal(2));
assertEquals(4, values.intVal(2));
assertEquals(4, values.shortVal(2));
assertEquals(4, values.byteVal(2));
assertEquals("4.0", values.strVal(2));
assertEquals(new Double(4), values.objectVal(2));
}
public void testRangeScorer() throws Exception {
Expression expr = JavascriptCompiler.compile("2*popularity");
SimpleBindings bindings = new SimpleBindings();
bindings.add(new SortField("popularity", SortField.Type.LONG));
ValueSource vs = expr.getValueSource(bindings);
assertEquals(1, reader.leaves().size());
AtomicReaderContext leaf = reader.leaves().get(0);
FunctionValues values = vs.getValues(new HashMap<String,Object>(), leaf);
// everything
ValueSourceScorer scorer = values.getRangeScorer(leaf.reader(), "4", "40", true, true);
assertEquals(-1, scorer.docID());
assertEquals(0, scorer.nextDoc());
assertEquals(1, scorer.nextDoc());
assertEquals(2, scorer.nextDoc());
assertEquals(DocIdSetIterator.NO_MORE_DOCS, scorer.nextDoc());
// just the first doc
scorer = values.getRangeScorer(leaf.reader(), "4", "40", false, false);
assertEquals(-1, scorer.docID());
assertEquals(0, scorer.nextDoc());
assertEquals(DocIdSetIterator.NO_MORE_DOCS, scorer.nextDoc());
}
public void testEquals() throws Exception {
Expression expr = JavascriptCompiler.compile("sqrt(a) + ln(b)");
SimpleBindings bindings = new SimpleBindings();
bindings.add(new SortField("a", SortField.Type.INT));
bindings.add(new SortField("b", SortField.Type.INT));
ValueSource vs1 = expr.getValueSource(bindings);
// same instance
assertEquals(vs1, vs1);
// null
assertFalse(vs1.equals(null));
// other object
assertFalse(vs1.equals("foobar"));
// same bindings and expression instances
ValueSource vs2 = expr.getValueSource(bindings);
assertEquals(vs1.hashCode(), vs2.hashCode());
assertEquals(vs1, vs2);
// equiv bindings (different instance)
SimpleBindings bindings2 = new SimpleBindings();
bindings2.add(new SortField("a", SortField.Type.INT));
bindings2.add(new SortField("b", SortField.Type.INT));
ValueSource vs3 = expr.getValueSource(bindings2);
assertEquals(vs1, vs3);
// different bindings (same names, different types)
SimpleBindings bindings3 = new SimpleBindings();
bindings3.add(new SortField("a", SortField.Type.LONG));
bindings3.add(new SortField("b", SortField.Type.INT));
ValueSource vs4 = expr.getValueSource(bindings3);
assertFalse(vs1.equals(vs4));
}
}

60
lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleDocValues.java
View File

@ -17,8 +17,10 @@ package org.apache.lucene.queries.function.docvalues;
* limitations under the License.
*/
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.ValueSourceScorer;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueDouble;
@ -80,6 +82,64 @@ public abstract class DoubleDocValues extends FunctionValues {
public String toString(int doc) {
return vs.description() + '=' + strVal(doc);
}
@Override
public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
double lower,upper;
if (lowerVal==null) {
lower = Double.NEGATIVE_INFINITY;
} else {
lower = Double.parseDouble(lowerVal);
}
if (upperVal==null) {
upper = Double.POSITIVE_INFINITY;
} else {
upper = Double.parseDouble(upperVal);
}
final double l = lower;
final double u = upper;
if (includeLower && includeUpper) {
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
double docVal = doubleVal(doc);
return docVal >= l && docVal <= u;
}
};
}
else if (includeLower && !includeUpper) {
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
double docVal = doubleVal(doc);
return docVal >= l && docVal < u;
}
};
}
else if (!includeLower && includeUpper) {
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
double docVal = doubleVal(doc);
return docVal > l && docVal <= u;
}
};
}
else {
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
double docVal = doubleVal(doc);
return docVal > l && docVal < u;
}
};
}
}
@Override
public ValueFiller getValueFiller() {

36
lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntDocValues.java
View File

@ -17,8 +17,10 @@ package org.apache.lucene.queries.function.docvalues;
* limitations under the License.
*/
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.ValueSourceScorer;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueInt;
@ -75,6 +77,40 @@ public abstract class IntDocValues extends FunctionValues {
public String toString(int doc) {
return vs.description() + '=' + strVal(doc);
}
@Override
public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
int lower,upper;
// instead of using separate comparison functions, adjust the endpoints.
if (lowerVal==null) {
lower = Integer.MIN_VALUE;
} else {
lower = Integer.parseInt(lowerVal);
if (!includeLower && lower < Integer.MAX_VALUE) lower++;
}
if (upperVal==null) {
upper = Integer.MAX_VALUE;
} else {
upper = Integer.parseInt(upperVal);
if (!includeUpper && upper > Integer.MIN_VALUE) upper--;
}
final int ll = lower;
final int uu = upper;
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
int val = intVal(doc);
// only check for deleted if it's the default value
// if (val==0 && reader.isDeleted(doc)) return false;
return val >= ll && val <= uu;
}
};
}
@Override
public ValueFiller getValueFiller() {

40
lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/LongDocValues.java
View File

@ -17,8 +17,10 @@ package org.apache.lucene.queries.function.docvalues;
* limitations under the License.
*/
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.ValueSourceScorer;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueLong;
@ -80,6 +82,44 @@ public abstract class LongDocValues extends FunctionValues {
public String toString(int doc) {
return vs.description() + '=' + strVal(doc);
}
protected long externalToLong(String extVal) {
return Long.parseLong(extVal);
}
@Override
public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
long lower,upper;
// instead of using separate comparison functions, adjust the endpoints.
if (lowerVal==null) {
lower = Long.MIN_VALUE;
} else {
lower = externalToLong(lowerVal);
if (!includeLower && lower < Long.MAX_VALUE) lower++;
}
if (upperVal==null) {
upper = Long.MAX_VALUE;
} else {
upper = externalToLong(upperVal);
if (!includeUpper && upper > Long.MIN_VALUE) upper--;
}
final long ll = lower;
final long uu = upper;
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
long val = longVal(doc);
// only check for deleted if it's the default value
// if (val==0 && reader.isDeleted(doc)) return false;
return val >= ll && val <= uu;
}
};
}
@Override
public ValueFiller getValueFiller() {

58
lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java
View File

@ -67,64 +67,6 @@ public class DoubleFieldSource extends FieldCacheSource {
return arr.get(doc) != 0 || valid.get(doc);
}
@Override
public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
double lower,upper;
if (lowerVal==null) {
lower = Double.NEGATIVE_INFINITY;
} else {
lower = Double.parseDouble(lowerVal);
}
if (upperVal==null) {
upper = Double.POSITIVE_INFINITY;
} else {
upper = Double.parseDouble(upperVal);
}
final double l = lower;
final double u = upper;
if (includeLower && includeUpper) {
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
double docVal = doubleVal(doc);
return docVal >= l && docVal <= u;
}
};
}
else if (includeLower && !includeUpper) {
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
double docVal = doubleVal(doc);
return docVal >= l && docVal < u;
}
};
}
else if (!includeLower && includeUpper) {
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
double docVal = doubleVal(doc);
return docVal > l && docVal <= u;
}
};
}
else {
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
double docVal = doubleVal(doc);
return docVal > l && docVal < u;
}
};
}
}
@Override
public ValueFiller getValueFiller() {
return new ValueFiller() {

34
lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java
View File

@ -100,40 +100,6 @@ public class IntFieldSource extends FieldCacheSource {
return description() + '=' + intVal(doc);
}
@Override
public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
int lower,upper;
// instead of using separate comparison functions, adjust the endpoints.
if (lowerVal==null) {
lower = Integer.MIN_VALUE;
} else {
lower = Integer.parseInt(lowerVal);
if (!includeLower && lower < Integer.MAX_VALUE) lower++;
}
if (upperVal==null) {
upper = Integer.MAX_VALUE;
} else {
upper = Integer.parseInt(upperVal);
if (!includeUpper && upper > Integer.MIN_VALUE) upper--;
}
final int ll = lower;
final int uu = upper;
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
int val = arr.get(doc);
// only check for deleted if it's the default value
// if (val==0 && reader.isDeleted(doc)) return false;
return val >= ll && val <= uu;
}
};
}
@Override
public ValueFiller getValueFiller() {
return new ValueFiller() {

33
lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java
View File

@ -91,37 +91,8 @@ public class LongFieldSource extends FieldCacheSource {
}
@Override
public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
long lower,upper;
// instead of using separate comparison functions, adjust the endpoints.
if (lowerVal==null) {
lower = Long.MIN_VALUE;
} else {
lower = externalToLong(lowerVal);
if (!includeLower && lower < Long.MAX_VALUE) lower++;
}
if (upperVal==null) {
upper = Long.MAX_VALUE;
} else {
upper = externalToLong(upperVal);
if (!includeUpper && upper > Long.MIN_VALUE) upper--;
}
final long ll = lower;
final long uu = upper;
return new ValueSourceScorer(reader, this) {
@Override
public boolean matchesValue(int doc) {
long val = arr.get(doc);
// only check for deleted if it's the default value
// if (val==0 && reader.isDeleted(doc)) return false;
return val >= ll && val <= uu;
}
};
protected long externalToLong(String extVal) {
return LongFieldSource.this.externalToLong(extVal);
}
@Override