Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-7609: Refactor expressions module to use DoubleValuesSource

This commit is contained in:
Alan Woodward 2017-01-05 13:15:19 +00:00
parent da30f21f5d
commit 8b055382d6
21 changed files with 203 additions and 515 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lucene/CHANGES.txt
View File

@ -74,6 +74,11 @@ API Changes
grouping Collectors are renamed to remove the Abstract* prefix.
(Alan Woodward, Martijn van Groningen)
* LUCENE-7609: The expressions module now uses the DoubleValuesSource API, and
no longer depends on the queries module. Expression#getValueSource() is
replaced with Expression#getDoubleValuesSource(). (Alan Woodward, Adrien
Grand)
New features
* LUCENE-5867: Added BooleanSimilarity. (Robert Muir, Adrien Grand)

2
lucene/demo/src/java/org/apache/lucene/demo/facet/ExpressionAggregationFacetsExample.java
View File

@ -103,7 +103,7 @@ public class ExpressionAggregationFacetsExample {
FacetsCollector.search(searcher, new MatchAllDocsQuery(), 10, fc);
// Retrieve results
Facets facets = new TaxonomyFacetSumValueSource(taxoReader, config, fc, expr.getValueSource(bindings));
Facets facets = new TaxonomyFacetSumValueSource(taxoReader, config, fc, expr.getDoubleValuesSource(bindings));
FacetResult result = facets.getTopChildren(10, "A");
indexReader.close();

11
lucene/expressions/build.xml
View File

@ -26,7 +26,6 @@
<path id="classpath">
<path refid="base.classpath"/>
<fileset dir="lib"/>
<pathelement path="${queries.jar}"/>
</path>
<path id="test.classpath">
@ -35,16 +34,6 @@
<pathelement path="src/test-files"/>
</path>
<target name="compile-core" depends="jar-queries,common.compile-core" />
<target name="javadocs" depends="javadocs-queries,compile-core,check-javadocs-uptodate" unless="javadocs-uptodate-${name}">
<invoke-module-javadoc>
<links>
<link href="../queries"/>
</links>
</invoke-module-javadoc>
</target>
<target name="regenerate" depends="run-antlr"/>
<target name="resolve-antlr" xmlns:ivy="antlib:org.apache.ivy.ant">

14
lucene/expressions/src/java/org/apache/lucene/expressions/Bindings.java
View File

@ -16,7 +16,7 @@
*/
package org.apache.lucene.expressions;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.DoubleValuesSource;
/**
* Binds variable names in expressions to actual data.
@ -31,14 +31,10 @@ public abstract class Bindings {
/** Sole constructor. (For invocation by subclass
* constructors, typically implicit.) */
protected Bindings() {}
/**
* Returns a ValueSource bound to the variable name.
* Returns a DoubleValuesSource bound to the variable name
*/
public abstract ValueSource getValueSource(String name);
/** Returns a {@code ValueSource} over relevance scores */
protected final ValueSource getScoreValueSource() {
return new ScoreValueSource();
}
public abstract DoubleValuesSource getDoubleValuesSource(String name);
}

21
lucene/expressions/src/java/org/apache/lucene/expressions/Expression.java
View File

@ -16,9 +16,9 @@
*/
package org.apache.lucene.expressions;
import org.apache.lucene.expressions.js.JavascriptCompiler; // javadocs
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.expressions.js.JavascriptCompiler;
import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.Rescorer;
import org.apache.lucene.search.SortField;
@ -63,26 +63,25 @@ public abstract class Expression {
}
/**
* Evaluates the expression for the given document.
* Evaluates the expression for the current document.
*
* @param document <code>docId</code> of the document to compute a value for
* @param functionValues {@link FunctionValues} for each element of {@link #variables}.
* @param functionValues {@link DoubleValues} for each element of {@link #variables}.
* @return The computed value of the expression for the given document.
*/
public abstract double evaluate(int document, FunctionValues[] functionValues);
public abstract double evaluate(DoubleValues[] functionValues);
/**
* Get a value source which can compute the value of this expression in the context of the given bindings.
* Get a DoubleValuesSource which can compute the value of this expression in the context of the given bindings.
* @param bindings Bindings to use for external values in this expression
* @return A value source which will evaluate this expression when used
* @return A DoubleValuesSource which will evaluate this expression when used
*/
public ValueSource getValueSource(Bindings bindings) {
public DoubleValuesSource getDoubleValuesSource(Bindings bindings) {
return new ExpressionValueSource(bindings, this);
}
/** Get a sort field which can be used to rank documents by this expression. */
public SortField getSortField(Bindings bindings, boolean reverse) {
return getValueSource(bindings).getSortField(reverse);
return getDoubleValuesSource(bindings).getSortField(reverse);
}
/** Get a {@link Rescorer}, to rescore first-pass hits

100
lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionComparator.java
View File

@ -1,100 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.expressions;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.LeafFieldComparator;
import org.apache.lucene.search.Scorer;
/** A custom comparator for sorting documents by an expression */
class ExpressionComparator extends FieldComparator<Double> implements LeafFieldComparator {
private final double[] values;
private double bottom;
private double topValue;
private ValueSource source;
private FunctionValues scores;
private LeafReaderContext readerContext;
public ExpressionComparator(ValueSource source, int numHits) {
values = new double[numHits];
this.source = source;
}
// TODO: change FieldComparator.setScorer to throw IOException and remove this try-catch
@Override
public void setScorer(Scorer scorer) {
// TODO: might be cleaner to lazy-init 'source' and set scorer after?
assert readerContext != null;
try {
Map<String,Object> context = new HashMap<>();
assert scorer != null;
context.put("scorer", scorer);
scores = source.getValues(context, readerContext);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public int compare(int slot1, int slot2) {
return Double.compare(values[slot1], values[slot2]);
}
@Override
public void setBottom(int slot) {
bottom = values[slot];
}
@Override
public void setTopValue(Double value) {
topValue = value.doubleValue();
}
@Override
public int compareBottom(int doc) throws IOException {
return Double.compare(bottom, scores.doubleVal(doc));
}
@Override
public void copy(int slot, int doc) throws IOException {
values[slot] = scores.doubleVal(doc);
}
@Override
public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException {
this.readerContext = context;
return this;
}
@Override
public Double value(int slot) {
return Double.valueOf(values[slot]);
}
@Override
public int compareTop(int doc) throws IOException {
return Double.compare(topValue, scores.doubleVal(doc));
}
}

35
lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionFunctionValues.java
View File

@ -16,20 +16,16 @@
*/
package org.apache.lucene.expressions;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
import java.io.IOException;
/** A {@link FunctionValues} which evaluates an expression */
class ExpressionFunctionValues extends DoubleDocValues {
import org.apache.lucene.search.DoubleValues;
/** A {@link DoubleValues} which evaluates an expression */
class ExpressionFunctionValues extends DoubleValues {
final Expression expression;
final FunctionValues[] functionValues;
final DoubleValues[] functionValues;
int currentDocument = -1;
double currentValue;
ExpressionFunctionValues(ValueSource parent, Expression expression, FunctionValues[] functionValues) {
super(parent);
ExpressionFunctionValues(Expression expression, DoubleValues[] functionValues) {
if (expression == null) {
throw new NullPointerException();
}
@ -39,14 +35,17 @@ class ExpressionFunctionValues extends DoubleDocValues {
this.expression = expression;
this.functionValues = functionValues;
}
@Override
public boolean advanceExact(int doc) throws IOException {
for (DoubleValues v : functionValues) {
v.advanceExact(doc);
}
return true;
}
@Override
public double doubleVal(int document) {
if (currentDocument != document) {
currentDocument = document;
currentValue = expression.evaluate(document, functionValues);
}
return currentValue;
public double doubleValue() {
return expression.evaluate(functionValues);
}
}

33
lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionRescorer.java
View File

@ -20,13 +20,11 @@ package org.apache.lucene.expressions;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Rescorer;
@ -49,7 +47,7 @@ class ExpressionRescorer extends SortRescorer {
private final Expression expression;
private final Bindings bindings;
/** Uses the provided {@link ValueSource} to assign second
/** Uses the provided {@link Expression} to assign second
* pass scores. */
public ExpressionRescorer(Expression expression, Bindings bindings) {
super(new Sort(expression.getSortField(bindings, true)));
@ -57,6 +55,21 @@ class ExpressionRescorer extends SortRescorer {
this.bindings = bindings;
}
private static DoubleValues scores(int doc, float score) {
return new DoubleValues() {
@Override
public double doubleValue() throws IOException {
return score;
}
@Override
public boolean advanceExact(int target) throws IOException {
assert doc == target;
return true;
}
};
}
@Override
public Explanation explain(IndexSearcher searcher, Explanation firstPassExplanation, int docID) throws IOException {
Explanation superExpl = super.explain(searcher, firstPassExplanation, docID);
@ -65,18 +78,14 @@ class ExpressionRescorer extends SortRescorer {
int subReader = ReaderUtil.subIndex(docID, leaves);
LeafReaderContext readerContext = leaves.get(subReader);
int docIDInSegment = docID - readerContext.docBase;
Map<String,Object> context = new HashMap<>();
FakeScorer fakeScorer = new FakeScorer();
fakeScorer.score = firstPassExplanation.getValue();
fakeScorer.doc = docIDInSegment;
context.put("scorer", fakeScorer);
DoubleValues scores = scores(docIDInSegment, firstPassExplanation.getValue());
List<Explanation> subs = new ArrayList<>(Arrays.asList(superExpl.getDetails()));
for(String variable : expression.variables) {
subs.add(Explanation.match((float) bindings.getValueSource(variable).getValues(context, readerContext).doubleVal(docIDInSegment),
"variable \"" + variable + "\""));
DoubleValues dv = bindings.getDoubleValuesSource(variable).getValues(readerContext, scores);
if (dv.advanceExact(docIDInSegment))
subs.add(Explanation.match((float) dv.doubleValue(), "variable \"" + variable + "\""));
}
return Explanation.match(superExpl.getValue(), superExpl.getDescription(), subs);

77
lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionSortField.java
View File

@ -1,77 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.expressions;
import java.io.IOException;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.SortField;
/** A {@link SortField} which sorts documents by the evaluated value of an expression for each document */
class ExpressionSortField extends SortField {
private final ExpressionValueSource source;
ExpressionSortField(String name, ExpressionValueSource source, boolean reverse) {
super(name, Type.CUSTOM, reverse);
this.source = source;
}
@Override
public FieldComparator<?> getComparator(final int numHits, final int sortPos) throws IOException {
return new ExpressionComparator(source, numHits);
}
@Override
public int hashCode() {
final int prime = 31;
int result = super.hashCode();
result = prime * result + ((source == null) ? 0 : source.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (!super.equals(obj)) return false;
if (getClass() != obj.getClass()) return false;
ExpressionSortField other = (ExpressionSortField) obj;
if (source == null) {
if (other.source != null) return false;
} else if (!source.equals(other.source)) return false;
return true;
}
@Override
public String toString() {
StringBuilder buffer = new StringBuilder();
buffer.append("<expr \"");
buffer.append(getField());
buffer.append("\">");
if (getReverse()) {
buffer.append('!');
}
return buffer.toString();
}
@Override
public boolean needsScores() {
return source.needsScores();
}
}

74
lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionValueSource.java
View File

@ -20,76 +20,77 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.search.DoubleValuesSource;
/**
* A {@link ValueSource} which evaluates a {@link Expression} given the context of an {@link Bindings}.
* A {@link DoubleValuesSource} which evaluates a {@link Expression} given the context of an {@link Bindings}.
*/
@SuppressWarnings({"rawtypes", "unchecked"})
final class ExpressionValueSource extends ValueSource {
final ValueSource variables[];
final class ExpressionValueSource extends DoubleValuesSource {
final DoubleValuesSource variables[];
final Expression expression;
final boolean needsScores;
ExpressionValueSource(Bindings bindings, Expression expression) {
if (bindings == null) throw new NullPointerException();
if (expression == null) throw new NullPointerException();
this.expression = expression;
variables = new ValueSource[expression.variables.length];
this.expression = Objects.requireNonNull(expression);
variables = new DoubleValuesSource[expression.variables.length];
boolean needsScores = false;
for (int i = 0; i < variables.length; i++) {
ValueSource source = bindings.getValueSource(expression.variables[i]);
if (source instanceof ScoreValueSource) {
needsScores = true;
} else if (source instanceof ExpressionValueSource) {
if (((ExpressionValueSource)source).needsScores()) {
needsScores = true;
}
} else if (source == null) {
DoubleValuesSource source = bindings.getDoubleValuesSource(expression.variables[i]);
if (source == null) {
throw new RuntimeException("Internal error. Variable (" + expression.variables[i] + ") does not exist.");
}
needsScores |= source.needsScores();
variables[i] = source;
}
this.needsScores = needsScores;
}
@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
Map<String, FunctionValues> valuesCache = (Map<String, FunctionValues>)context.get("valuesCache");
if (valuesCache == null) {
valuesCache = new HashMap<>();
context = new HashMap(context);
context.put("valuesCache", valuesCache);
}
FunctionValues[] externalValues = new FunctionValues[expression.variables.length];
public DoubleValues getValues(LeafReaderContext readerContext, DoubleValues scores) throws IOException {
Map<String, DoubleValues> valuesCache = new HashMap<>();
DoubleValues[] externalValues = new DoubleValues[expression.variables.length];
for (int i = 0; i < variables.length; ++i) {
String externalName = expression.variables[i];
FunctionValues values = valuesCache.get(externalName);
DoubleValues values = valuesCache.get(externalName);
if (values == null) {
values = variables[i].getValues(context, readerContext);
values = variables[i].getValues(readerContext, scores);
if (values == null) {
throw new RuntimeException("Internal error. External (" + externalName + ") does not exist.");
}
valuesCache.put(externalName, values);
}
externalValues[i] = values;
externalValues[i] = zeroWhenUnpositioned(values);
}
return new ExpressionFunctionValues(this, expression, externalValues);
return new ExpressionFunctionValues(expression, externalValues);
}
private static DoubleValues zeroWhenUnpositioned(DoubleValues in) {
return new DoubleValues() {
boolean positioned = false;
@Override
public double doubleValue() throws IOException {
return positioned ? in.doubleValue() : 0;
}
@Override
public boolean advanceExact(int doc) throws IOException {
return positioned = in.advanceExact(doc);
}
};
}
@Override
public SortField getSortField(boolean reverse) {
return new ExpressionSortField(expression.sourceText, this, reverse);
}
@Override
public String description() {
public String toString() {
return "expr(" + expression.sourceText + ")";
}
@ -132,7 +133,8 @@ final class ExpressionValueSource extends ValueSource {
return true;
}
boolean needsScores() {
@Override
public boolean needsScores() {
return needsScores;
}
}

46
lucene/expressions/src/java/org/apache/lucene/expressions/ScoreFunctionValues.java
View File

@ -1,46 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.expressions;
import java.io.IOException;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
import org.apache.lucene.search.Scorer;
/**
* A utility class to allow expressions to access the score as a {@link FunctionValues}.
*/
class ScoreFunctionValues extends DoubleDocValues {
final Scorer scorer;
ScoreFunctionValues(ValueSource parent, Scorer scorer) {
super(parent);
this.scorer = scorer;
}
@Override
public double doubleVal(int document) {
try {
assert document == scorer.docID();
return scorer.score();
} catch (IOException exception) {
throw new RuntimeException(exception);
}
}
}

61
lucene/expressions/src/java/org/apache/lucene/expressions/ScoreValueSource.java
View File

@ -1,61 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.expressions;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.Scorer;
import java.io.IOException;
import java.util.Map;
/**
* A {@link ValueSource} which uses the {@link Scorer} passed through
* the context map by {@link ExpressionComparator}.
*/
@SuppressWarnings({"rawtypes"})
class ScoreValueSource extends ValueSource {
/**
* <code>context</code> must contain a key "scorer" which is a {@link Scorer}.
*/
@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
Scorer v = (Scorer) context.get("scorer");
if (v == null) {
throw new IllegalStateException("Expressions referencing the score can only be used for sorting");
}
return new ScoreFunctionValues(this, v);
}
@Override
public boolean equals(Object o) {
return o == this;
}
@Override
public int hashCode() {
return System.identityHashCode(this);
}
@Override
public String description() {
return "score()";
}
}

30
lucene/expressions/src/java/org/apache/lucene/expressions/SimpleBindings.java
View File

@ -20,11 +20,7 @@ package org.apache.lucene.expressions;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.DoubleFieldSource;
import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
import org.apache.lucene.queries.function.valuesource.IntFieldSource;
import org.apache.lucene.queries.function.valuesource.LongFieldSource;
import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.SortField;
/**
@ -64,9 +60,9 @@ public final class SimpleBindings extends Bindings {
}
/**
* Bind a {@link ValueSource} directly to the given name.
* Bind a {@link DoubleValuesSource} directly to the given name.
*/
public void add(String name, ValueSource source) { map.put(name, source); }
public void add(String name, DoubleValuesSource source) { map.put(name, source); }
/**
* Adds an Expression to the bindings.
@ -78,27 +74,27 @@ public final class SimpleBindings extends Bindings {
}
@Override
public ValueSource getValueSource(String name) {
public DoubleValuesSource getDoubleValuesSource(String name) {
Object o = map.get(name);
if (o == null) {
throw new IllegalArgumentException("Invalid reference '" + name + "'");
} else if (o instanceof Expression) {
return ((Expression)o).getValueSource(this);
} else if (o instanceof ValueSource) {
return ((ValueSource)o);
return ((Expression)o).getDoubleValuesSource(this);
} else if (o instanceof DoubleValuesSource) {
return ((DoubleValuesSource) o);
}
SortField field = (SortField) o;
switch(field.getType()) {
case INT:
return new IntFieldSource(field.getField());
return DoubleValuesSource.fromIntField(field.getField());
case LONG:
return new LongFieldSource(field.getField());
return DoubleValuesSource.fromLongField(field.getField());
case FLOAT:
return new FloatFieldSource(field.getField());
return DoubleValuesSource.fromFloatField(field.getField());
case DOUBLE:
return new DoubleFieldSource(field.getField());
return DoubleValuesSource.fromDoubleField(field.getField());
case SCORE:
return getScoreValueSource();
return DoubleValuesSource.SCORES;
default:
throw new UnsupportedOperationException();
}
@ -113,7 +109,7 @@ public final class SimpleBindings extends Bindings {
if (o instanceof Expression) {
Expression expr = (Expression) o;
try {
expr.getValueSource(this);
expr.getDoubleValuesSource(this);
} catch (StackOverflowError e) {
throw new IllegalArgumentException("Recursion Error: Cycle detected originating in (" + expr.sourceText + ")");
}

15
lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
View File

@ -39,7 +39,7 @@ import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.ParseTree;
import org.apache.lucene.expressions.Expression;
import org.apache.lucene.expressions.js.JavascriptParser.ExpressionContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.util.IOUtils;
import org.objectweb.asm.ClassWriter;
import org.objectweb.asm.Label;
@ -93,13 +93,13 @@ public final class JavascriptCompiler {
private static final String COMPILED_EXPRESSION_INTERNAL = COMPILED_EXPRESSION_CLASS.replace('.', '/');
static final Type EXPRESSION_TYPE = Type.getType(Expression.class);
static final Type FUNCTION_VALUES_TYPE = Type.getType(FunctionValues.class);
static final Type FUNCTION_VALUES_TYPE = Type.getType(DoubleValues.class);
private static final org.objectweb.asm.commons.Method
EXPRESSION_CTOR = getAsmMethod(void.class, "<init>", String.class, String[].class),
EVALUATE_METHOD = getAsmMethod(double.class, "evaluate", int.class, FunctionValues[].class);
EVALUATE_METHOD = getAsmMethod(double.class, "evaluate", DoubleValues[].class);
static final org.objectweb.asm.commons.Method DOUBLE_VAL_METHOD = getAsmMethod(double.class, "doubleVal", int.class);
static final org.objectweb.asm.commons.Method DOUBLE_VAL_METHOD = getAsmMethod(double.class, "doubleValue");
/** create an ASM Method object from return type, method name, and parameters. */
private static org.objectweb.asm.commons.Method getAsmMethod(Class<?> rtype, String name, Class<?>... ptypes) {
@ -155,8 +155,8 @@ public final class JavascriptCompiler {
*/
@SuppressWarnings({"unused", "null"})
private static void unusedTestCompile() throws IOException {
FunctionValues f = null;
double ret = f.doubleVal(2);
DoubleValues f = null;
double ret = f.doubleValue();
}
/**
@ -325,10 +325,9 @@ public final class JavascriptCompiler {
externalsMap.put(text, index);
}
gen.loadArg(1);
gen.loadArg(0);
gen.push(index);
gen.arrayLoad(FUNCTION_VALUES_TYPE);
gen.loadArg(0);
gen.invokeVirtual(FUNCTION_VALUES_TYPE, DOUBLE_VAL_METHOD);
gen.cast(Type.DOUBLE_TYPE, typeStack.peek());
} else {

5
lucene/expressions/src/java/org/apache/lucene/expressions/package-info.java
View File

@ -25,11 +25,12 @@
*
* <p>
* {@link org.apache.lucene.expressions.Bindings} - abstraction for binding external variables
* to a way to get a value for those variables for a particular document (ValueSource).
* to a way to get a value for those variables for a particular document (DoubleValuesSource).
* </p>
*
* <p>
* {@link org.apache.lucene.expressions.SimpleBindings} - default implementation of bindings which provide easy ways to bind sort fields and other expressions to external variables
* {@link org.apache.lucene.expressions.SimpleBindings} - default implementation of bindings which
* provide easy ways to bind sort fields and other expressions to external variables
* </p>
*/
package org.apache.lucene.expressions;

44
lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java
View File

@ -16,18 +16,20 @@
*/
package org.apache.lucene.expressions;
import java.io.IOException;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.expressions.js.JavascriptCompiler;
import org.apache.lucene.expressions.js.VariableContext;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource;
import org.apache.lucene.queries.function.valuesource.IntFieldSource;
import org.apache.lucene.search.CheckHits;
import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
@ -39,9 +41,9 @@ import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import static org.apache.lucene.expressions.js.VariableContext.Type.INT_INDEX;
import static org.apache.lucene.expressions.js.VariableContext.Type.MEMBER;
import static org.apache.lucene.expressions.js.VariableContext.Type.STR_INDEX;
import static org.apache.lucene.expressions.js.VariableContext.Type.INT_INDEX;
/** simple demo of using expressions */
@ -236,7 +238,7 @@ public class TestDemoExpressions extends LuceneTestCase {
public void testStaticExtendedVariableExample() throws Exception {
Expression popularity = JavascriptCompiler.compile("doc[\"popularity\"].value");
SimpleBindings bindings = new SimpleBindings();
bindings.add("doc['popularity'].value", new IntFieldSource("popularity"));
bindings.add("doc['popularity'].value", DoubleValuesSource.fromIntField("popularity"));
Sort sort = new Sort(popularity.getSortField(bindings, true));
TopFieldDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort);
@ -250,6 +252,30 @@ public class TestDemoExpressions extends LuceneTestCase {
assertEquals(2D, (Double)d.fields[0], 1E-4);
}
private static DoubleValuesSource constant(double value) {
return new DoubleValuesSource() {
@Override
public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
return new DoubleValues() {
@Override
public double doubleValue() throws IOException {
return value;
}
@Override
public boolean advanceExact(int doc) throws IOException {
return true;
}
};
}
@Override
public boolean needsScores() {
return false;
}
};
}
public void testDynamicExtendedVariableExample() throws Exception {
Expression popularity = JavascriptCompiler.compile("doc['popularity'].value + magicarray[0] + fourtytwo");
@ -258,7 +284,7 @@ public class TestDemoExpressions extends LuceneTestCase {
// filled in with proper error messages for a real use case.
Bindings bindings = new Bindings() {
@Override
public ValueSource getValueSource(String name) {
public DoubleValuesSource getDoubleValuesSource(String name) {
VariableContext[] var = VariableContext.parse(name);
assert var[0].type == MEMBER;
String base = var[0].text;
@ -266,7 +292,7 @@ public class TestDemoExpressions extends LuceneTestCase {
if (var.length > 1 && var[1].type == STR_INDEX) {
String field = var[1].text;
if (var.length > 2 && var[2].type == MEMBER && var[2].text.equals("value")) {
return new IntFieldSource(field);
return DoubleValuesSource.fromIntField(field);
} else {
fail("member: " + var[2].text);// error case, non/missing "value" member access
}
@ -275,12 +301,12 @@ public class TestDemoExpressions extends LuceneTestCase {
}
} else if (base.equals("magicarray")) {
if (var.length > 1 && var[1].type == INT_INDEX) {
return new DoubleConstValueSource(2048);
return constant(2048);
} else {
fail();// error case, magic array isn't an array
}
} else if (base.equals("fourtytwo")) {
return new DoubleConstValueSource(42);
return constant(42);
} else {
fail();// error case (variable doesn't exist)
}

2
lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionSortField.java
View File

@ -31,7 +31,7 @@ public class TestExpressionSortField extends LuceneTestCase {
bindings.add(new SortField("popularity", SortField.Type.INT));
SortField sf = expr.getSortField(bindings, true);
assertEquals("<expr \"sqrt(_score) + ln(popularity)\">!", sf.toString());
assertEquals("<expr(sqrt(_score) + ln(popularity))>!", sf.toString());
}
public void testEquals() throws Exception {

119
lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionValueSource.java
View File

@ -17,21 +17,17 @@
package org.apache.lucene.expressions;
import java.util.HashMap;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.expressions.js.JavascriptCompiler;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.ValueSourceScorer;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.SortField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@ -47,7 +43,7 @@ public class TestExpressionValueSource extends LuceneTestCase {
IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
iwc.setMergePolicy(newLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
Document doc = new Document();
doc.add(newStringField("id", "1", Field.Store.YES));
doc.add(newTextField("body", "some contents and more contents", Field.Store.NO));
@ -58,6 +54,7 @@ public class TestExpressionValueSource extends LuceneTestCase {
doc.add(newStringField("id", "2", Field.Store.YES));
doc.add(newTextField("body", "another document with different contents", Field.Store.NO));
doc.add(new NumericDocValuesField("popularity", 20));
doc.add(new NumericDocValuesField("count", 1));
iw.addDocument(doc);
doc = new Document();
@ -77,81 +74,34 @@ public class TestExpressionValueSource extends LuceneTestCase {
dir.close();
super.tearDown();
}
public void testTypes() throws Exception {
Expression expr = JavascriptCompiler.compile("2*popularity");
SimpleBindings bindings = new SimpleBindings();
bindings.add(new SortField("popularity", SortField.Type.LONG));
ValueSource vs = expr.getValueSource(bindings);
assertEquals(1, reader.leaves().size());
LeafReaderContext leaf = reader.leaves().get(0);
FunctionValues values = vs.getValues(new HashMap<String,Object>(), leaf);
assertEquals(10, values.doubleVal(0), 0);
assertEquals(10, values.floatVal(0), 0);
assertEquals(10, values.longVal(0));
assertEquals(10, values.intVal(0));
assertEquals(10, values.shortVal(0));
assertEquals(10, values.byteVal(0));
assertEquals("10.0", values.strVal(0));
assertEquals(new Double(10), values.objectVal(0));
assertEquals(40, values.doubleVal(1), 0);
assertEquals(40, values.floatVal(1), 0);
assertEquals(40, values.longVal(1));
assertEquals(40, values.intVal(1));
assertEquals(40, values.shortVal(1));
assertEquals(40, values.byteVal(1));
assertEquals("40.0", values.strVal(1));
assertEquals(new Double(40), values.objectVal(1));
assertEquals(4, values.doubleVal(2), 0);
assertEquals(4, values.floatVal(2), 0);
assertEquals(4, values.longVal(2));
assertEquals(4, values.intVal(2));
assertEquals(4, values.shortVal(2));
assertEquals(4, values.byteVal(2));
assertEquals("4.0", values.strVal(2));
assertEquals(new Double(4), values.objectVal(2));
}
public void testRangeScorer() throws Exception {
Expression expr = JavascriptCompiler.compile("2*popularity");
SimpleBindings bindings = new SimpleBindings();
bindings.add(new SortField("popularity", SortField.Type.LONG));
ValueSource vs = expr.getValueSource(bindings);
assertEquals(1, reader.leaves().size());
LeafReaderContext leaf = reader.leaves().get(0);
FunctionValues values = vs.getValues(new HashMap<String,Object>(), leaf);
// everything
ValueSourceScorer scorer = values.getRangeScorer(leaf, "4", "40", true, true);
DocIdSetIterator iter = scorer.iterator();
assertEquals(-1, iter.docID());
assertEquals(0, iter.nextDoc());
assertEquals(1, iter.nextDoc());
assertEquals(2, iter.nextDoc());
assertEquals(DocIdSetIterator.NO_MORE_DOCS, iter.nextDoc());
// just the first doc
values = vs.getValues(new HashMap<String,Object>(), leaf);
scorer = values.getRangeScorer(leaf, "4", "40", false, false);
iter = scorer.iterator();
assertEquals(-1, scorer.docID());
assertEquals(0, iter.nextDoc());
assertEquals(DocIdSetIterator.NO_MORE_DOCS, iter.nextDoc());
public void testDoubleValuesSourceTypes() throws Exception {
Expression expr = JavascriptCompiler.compile("2*popularity + count");
SimpleBindings bindings = new SimpleBindings();
bindings.add(new SortField("popularity", SortField.Type.LONG));
bindings.add(new SortField("count", SortField.Type.LONG));
DoubleValuesSource vs = expr.getDoubleValuesSource(bindings);
assertEquals(1, reader.leaves().size());
LeafReaderContext leaf = reader.leaves().get(0);
DoubleValues values = vs.getValues(leaf, null);
assertTrue(values.advanceExact(0));
assertEquals(10, values.doubleValue(), 0);
assertTrue(values.advanceExact(1));
assertEquals(41, values.doubleValue(), 0);
assertTrue(values.advanceExact(2));
assertEquals(4, values.doubleValue(), 0);
}
public void testEquals() throws Exception {
public void testDoubleValuesSourceEquals() throws Exception {
Expression expr = JavascriptCompiler.compile("sqrt(a) + ln(b)");
SimpleBindings bindings = new SimpleBindings();
SimpleBindings bindings = new SimpleBindings();
bindings.add(new SortField("a", SortField.Type.INT));
bindings.add(new SortField("b", SortField.Type.INT));
ValueSource vs1 = expr.getValueSource(bindings);
DoubleValuesSource vs1 = expr.getDoubleValuesSource(bindings);
// same instance
assertEquals(vs1, vs1);
// null
@ -159,20 +109,21 @@ public class TestExpressionValueSource extends LuceneTestCase {
// other object
assertFalse(vs1.equals("foobar"));
// same bindings and expression instances
ValueSource vs2 = expr.getValueSource(bindings);
DoubleValuesSource vs2 = expr.getDoubleValuesSource(bindings);
assertEquals(vs1.hashCode(), vs2.hashCode());
assertEquals(vs1, vs2);
// equiv bindings (different instance)
SimpleBindings bindings2 = new SimpleBindings();
SimpleBindings bindings2 = new SimpleBindings();
bindings2.add(new SortField("a", SortField.Type.INT));
bindings2.add(new SortField("b", SortField.Type.INT));
ValueSource vs3 = expr.getValueSource(bindings2);
DoubleValuesSource vs3 = expr.getDoubleValuesSource(bindings2);
assertEquals(vs1, vs3);
// different bindings (same names, different types)
SimpleBindings bindings3 = new SimpleBindings();
SimpleBindings bindings3 = new SimpleBindings();
bindings3.add(new SortField("a", SortField.Type.LONG));
bindings3.add(new SortField("b", SortField.Type.INT));
ValueSource vs4 = expr.getValueSource(bindings3);
bindings3.add(new SortField("b", SortField.Type.FLOAT));
DoubleValuesSource vs4 = expr.getDoubleValuesSource(bindings3);
assertFalse(vs1.equals(vs4));
}
}

20
lucene/expressions/src/test/org/apache/lucene/expressions/js/TestCustomFunctions.java
View File

@ -50,7 +50,7 @@ public class TestCustomFunctions extends LuceneTestCase {
public void testDefaultList() throws Exception {
Map<String,Method> functions = JavascriptCompiler.DEFAULT_FUNCTIONS;
Expression expr = JavascriptCompiler.compile("sqrt(20)", functions, getClass().getClassLoader());
assertEquals(Math.sqrt(20), expr.evaluate(0, null), DELTA);
assertEquals(Math.sqrt(20), expr.evaluate(null), DELTA);
}
public static double zeroArgMethod() { return 5; }
@ -60,7 +60,7 @@ public class TestCustomFunctions extends LuceneTestCase {
Map<String,Method> functions = new HashMap<>();
functions.put("foo", getClass().getMethod("zeroArgMethod"));
Expression expr = JavascriptCompiler.compile("foo()", functions, getClass().getClassLoader());
assertEquals(5, expr.evaluate(0, null), DELTA);
assertEquals(5, expr.evaluate(null), DELTA);
}
public static double oneArgMethod(double arg1) { return 3 + arg1; }
@ -70,7 +70,7 @@ public class TestCustomFunctions extends LuceneTestCase {
Map<String,Method> functions = new HashMap<>();
functions.put("foo", getClass().getMethod("oneArgMethod", double.class));
Expression expr = JavascriptCompiler.compile("foo(3)", functions, getClass().getClassLoader());
assertEquals(6, expr.evaluate(0, null), DELTA);
assertEquals(6, expr.evaluate(null), DELTA);
}
public static double threeArgMethod(double arg1, double arg2, double arg3) { return arg1 + arg2 + arg3; }
@ -80,7 +80,7 @@ public class TestCustomFunctions extends LuceneTestCase {
Map<String,Method> functions = new HashMap<>();
functions.put("foo", getClass().getMethod("threeArgMethod", double.class, double.class, double.class));
Expression expr = JavascriptCompiler.compile("foo(3, 4, 5)", functions, getClass().getClassLoader());
assertEquals(12, expr.evaluate(0, null), DELTA);
assertEquals(12, expr.evaluate(null), DELTA);
}
/** tests a map with 2 functions */
@ -89,7 +89,7 @@ public class TestCustomFunctions extends LuceneTestCase {
functions.put("foo", getClass().getMethod("zeroArgMethod"));
functions.put("bar", getClass().getMethod("oneArgMethod", double.class));
Expression expr = JavascriptCompiler.compile("foo() + bar(3)", functions, getClass().getClassLoader());
assertEquals(11, expr.evaluate(0, null), DELTA);
assertEquals(11, expr.evaluate(null), DELTA);
}
/** tests invalid methods that are not allowed to become variables to be mapped */
@ -220,7 +220,7 @@ public class TestCustomFunctions extends LuceneTestCase {
// this should pass:
Expression expr = JavascriptCompiler.compile("bar()", functions, childLoader);
assertEquals(2.0, expr.evaluate(0, null), DELTA);
assertEquals(2.0, expr.evaluate(null), DELTA);
// use our classloader, not the foreign one, which should fail!
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
@ -232,9 +232,9 @@ public class TestCustomFunctions extends LuceneTestCase {
Map<String,Method> mixedFunctions = new HashMap<>(JavascriptCompiler.DEFAULT_FUNCTIONS);
mixedFunctions.putAll(functions);
expr = JavascriptCompiler.compile("bar()", mixedFunctions, childLoader);
assertEquals(2.0, expr.evaluate(0, null), DELTA);
assertEquals(2.0, expr.evaluate(null), DELTA);
expr = JavascriptCompiler.compile("sqrt(20)", mixedFunctions, childLoader);
assertEquals(Math.sqrt(20), expr.evaluate(0, null), DELTA);
assertEquals(Math.sqrt(20), expr.evaluate(null), DELTA);
// use our classloader, not the foreign one, which should fail!
expected = expectThrows(IllegalArgumentException.class, () -> {
@ -256,7 +256,7 @@ public class TestCustomFunctions extends LuceneTestCase {
String source = "3 * foo() / 5";
Expression expr = JavascriptCompiler.compile(source, functions, getClass().getClassLoader());
ArithmeticException expected = expectThrows(ArithmeticException.class, () -> {
expr.evaluate(0, null);
expr.evaluate(null);
});
assertEquals(MESSAGE, expected.getMessage());
StringWriter sw = new StringWriter();
@ -272,6 +272,6 @@ public class TestCustomFunctions extends LuceneTestCase {
functions.put("foo.bar", getClass().getMethod("zeroArgMethod"));
String source = "foo.bar()";
Expression expr = JavascriptCompiler.compile(source, functions, getClass().getClassLoader());
assertEquals(5, expr.evaluate(0, null), DELTA);
assertEquals(5, expr.evaluate(null), DELTA);
}
}

2
lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptFunction.java
View File

@ -24,7 +24,7 @@ public class TestJavascriptFunction extends LuceneTestCase {
private void assertEvaluatesTo(String expression, double expected) throws Exception {
Expression evaluator = JavascriptCompiler.compile(expression);
double actual = evaluator.evaluate(0, null);
double actual = evaluator.evaluate(null);
assertEquals(expected, actual, DELTA);
}

2
lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptOperations.java
View File

@ -22,7 +22,7 @@ import org.apache.lucene.util.LuceneTestCase;
public class TestJavascriptOperations extends LuceneTestCase {
private void assertEvaluatesTo(String expression, long expected) throws Exception {
Expression evaluator = JavascriptCompiler.compile(expression);
long actual = (long)evaluator.evaluate(0, null);
long actual = (long)evaluator.evaluate(null);
assertEquals(expected, actual);
}