honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Upgrade to Lucene 6.4.0 (#22724) 

* Upgrade to Lucene 6.4.0

`ValueSource`s are now converted to `DoubleValueSource`s using the Lucene adapter made for the migration to the new API in 6.4.0.
This commit is contained in:
Jim Ferenczi 2017-01-21 04:48:01 +01:00 committed by GitHub
parent cfb415de7b
commit 8028578305
53 changed files with 104 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
buildSrc/version.properties
View File

@ -1,5 +1,5 @@
elasticsearch = 6.0.0-alpha1
lucene = 6.4.0-snapshot-084f7a0
lucene = 6.4.0
# optional dependencies
spatial4j = 0.6

1
core/licenses/lucene-analyzers-common-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
ad1553dd2eed3a7cd5778bc7520821ac926b56df

1
core/licenses/lucene-analyzers-common-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
e0feb9281a7da7a7df62398ab0fc655d51f68fed

1
core/licenses/lucene-backward-codecs-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
dde630b1d09ff928a1f358951747cfad5c46b334

1
core/licenses/lucene-backward-codecs-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
14698ecbca1437615ee31d412d0edd3440b4fccf

1
core/licenses/lucene-core-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
1789bff323a0c013b126f4e51f1f269ebc631277

1
core/licenses/lucene-core-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
09dd516b847dcaf8da4e9096bf3189b0b3607aef

1
core/licenses/lucene-grouping-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
8cb17916d0e63705f1f715fe0d03ed32916a077a

1
core/licenses/lucene-grouping-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
68a8f986a0076ad784cbb20813b9465b94e4c846

1
core/licenses/lucene-highlighter-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
79d6ba8fa629a52ad3eb829d085836f5fd2f7a87

1
core/licenses/lucene-highlighter-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
6d921c1242b608a4dcd0784e32bcd94097ad93cd

1
core/licenses/lucene-join-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
19794d8f15402c991d9533bfcd67e2e7a34677ef

1
core/licenses/lucene-join-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
74d3cdf1bc863e3836b06f1865c970127cc15f26

1
core/licenses/lucene-memory-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
33e42d3019e072752258bd778912c8d4365470a1

1
core/licenses/lucene-memory-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
dd13729c0b401e3df11bce0c343d1e00f07b9a19

1
core/licenses/lucene-misc-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
a1b3271b3800da349c8b98f7b1a25b2b6192252a

1
core/licenses/lucene-misc-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
ce27abe3490bb8ccbebd2eefcb68f42a609ca986

1
core/licenses/lucene-queries-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
792716d805fcc5091931874c2f2f86f35da8b401

1
core/licenses/lucene-queries-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
bd1978e3fdac2fadf1068828b0b1b534a56873c3

1
core/licenses/lucene-queryparser-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
c3f8bbc6ebe8d31da41fcdb1fa73f13d8170ee62

1
core/licenses/lucene-queryparser-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
fb8fe41948fccf13b5dbb5d50441cac974544ade

1
core/licenses/lucene-sandbox-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
263901a19686c6cce7dd5c32a4934c42c62454dc

1
core/licenses/lucene-sandbox-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
e7f7d1ad298c4af264199d9199f34f2e4d9ca2b5

1
core/licenses/lucene-spatial-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
85426164fcc264a7e3bacc1a70602513540a261a

1
core/licenses/lucene-spatial-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
5d4b3ce4df83d0509e0b5f7eecda72af458ba225

1
core/licenses/lucene-spatial-extras-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
332cbfaa6b1ee0bf4d820018872988e15cd413d2

1
core/licenses/lucene-spatial-extras-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
3b486b51d3aede074ab6de890b427379d40c0438

1
core/licenses/lucene-spatial3d-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
3fe3e902b971f4aa2b4a3a417ba5dcf83e968428

1
core/licenses/lucene-spatial3d-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
344097014aeaaa0f94a217f3697e14ceee06581f

1
core/licenses/lucene-suggest-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
c4863fe45853163abfbe5c8b8bd7bdcf9a9c7b40

1
core/licenses/lucene-suggest-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
6c4706b86718f2653120e0dbfd24e03248dd2ea7

4
core/src/main/resources/org/elasticsearch/bootstrap/security.policy
View File

@ -31,7 +31,7 @@ grant codeBase "${codebase.securesm-1.1.jar}" {
//// Very special jar permissions:
//// These are dangerous permissions that we don't want to grant to everything.
grant codeBase "${codebase.lucene-core-6.4.0-snapshot-084f7a0.jar}" {
grant codeBase "${codebase.lucene-core-6.4.0.jar}" {
// needed to allow MMapDirectory's "unmap hack" (die unmap hack, die)
// java 8 package
permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
@ -42,7 +42,7 @@ grant codeBase "${codebase.lucene-core-6.4.0-snapshot-084f7a0.jar}" {
permission java.lang.RuntimePermission "accessDeclaredMembers";
};
grant codeBase "${codebase.lucene-misc-6.4.0-snapshot-084f7a0.jar}" {
grant codeBase "${codebase.lucene-misc-6.4.0.jar}" {
// needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper
permission java.nio.file.LinkPermission "hard";
};

2
core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
View File

@ -33,7 +33,7 @@ grant codeBase "${codebase.securemock-1.2.jar}" {
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
};
grant codeBase "${codebase.lucene-test-framework-6.4.0-snapshot-084f7a0.jar}" {
grant codeBase "${codebase.lucene-test-framework-6.4.0.jar}" {
// needed by RamUsageTester
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
// needed for testing hardlinks in StoreRecoveryTests since we install MockFS

1
modules/lang-expression/licenses/lucene-expressions-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
1f0be480db66169f45a9f0982fbad9f549b88b55

1
modules/lang-expression/licenses/lucene-expressions-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
09a755871c89fec98d3ca8372f283c1639dcadaf

19
modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java
View File

@ -19,25 +19,22 @@
package org.elasticsearch.script.expression;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.expressions.Expression;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.script.GeneralScriptException;
import java.util.HashMap;
import java.util.Map;
/**
* A bridge to evaluate an {@link Expression} against a map of variables in the context
* of an {@link ExecutableScript}.
*/
public class ExpressionExecutableScript implements ExecutableScript {
private final int NO_DOCUMENT = -1;
public final CompiledScript compiledScript;
public final Map<String, ReplaceableConstFunctionValues> functionValuesMap;
public final ReplaceableConstFunctionValues[] functionValuesArray;
public final Map<String, ReplaceableConstDoubleValues> functionValuesMap;
public final ReplaceableConstDoubleValues[] functionValuesArray;
public ExpressionExecutableScript(CompiledScript compiledScript, Map<String, Object> vars) {
this.compiledScript = compiledScript;
@ -51,12 +48,12 @@ public class ExpressionExecutableScript implements ExecutableScript {
" [" + vars.size() + "].");
}
functionValuesArray = new ReplaceableConstFunctionValues[functionValuesLength];
functionValuesArray = new ReplaceableConstDoubleValues[functionValuesLength];
functionValuesMap = new HashMap<>();
for (int functionValuesIndex = 0; functionValuesIndex < functionValuesLength; ++functionValuesIndex) {
String variableName = expression.variables[functionValuesIndex];
functionValuesArray[functionValuesIndex] = new ReplaceableConstFunctionValues();
functionValuesArray[functionValuesIndex] = new ReplaceableConstDoubleValues();
functionValuesMap.put(variableName, functionValuesArray[functionValuesIndex]);
}
@ -85,7 +82,7 @@ public class ExpressionExecutableScript implements ExecutableScript {
@Override
public Object run() {
try {
return ((Expression) compiledScript.compiled()).evaluate(NO_DOCUMENT, functionValuesArray);
return ((Expression) compiledScript.compiled()).evaluate(functionValuesArray);
} catch (Exception exception) {
throw new GeneralScriptException("Error evaluating " + compiledScript, exception);
}

15
modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
View File

@ -117,14 +117,15 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
// NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings,
// instead of complicating SimpleBindings (which should stay simple)
SimpleBindings bindings = new SimpleBindings();
ReplaceableConstValueSource specialValue = null;
ReplaceableConstDoubleValueSource specialValue = null;
boolean needsScores = false;
for (String variable : expr.variables) {
try {
if (variable.equals("_score")) {
bindings.add(new SortField("_score", SortField.Type.SCORE));
needsScores = true;
} else if (variable.equals("_value")) {
specialValue = new ReplaceableConstValueSource();
specialValue = new ReplaceableConstDoubleValueSource();
bindings.add("_value", specialValue);
// noop: _value is special for aggregations, and is handled in ExpressionScriptBindings
// TODO: if some uses it in a scoring expression, they will get a nasty failure when evaluating...need a
@ -136,7 +137,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
// but if we were to reverse it, we could provide a way to supply dynamic defaults for documents missing the field?
Object value = vars.get(variable);
if (value instanceof Number) {
bindings.add(variable, new DoubleConstValueSource(((Number) value).doubleValue()));
bindings.add(variable, new DoubleConstValueSource(((Number) value).doubleValue()).asDoubleValuesSource());
} else {
throw new ParseException("Parameter [" + variable + "] must be a numeric type", 0);
}
@ -225,16 +226,14 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
} else {
throw new ParseException("Field [" + fieldname + "] must be numeric, date, or geopoint", 5);
}
bindings.add(variable, valueSource);
needsScores |= valueSource.getSortField(false).needsScores();
bindings.add(variable, valueSource.asDoubleValuesSource());
}
} catch (Exception e) {
// we defer "binding" of variables until here: give context for that variable
throw convertToScriptException("link error", expr.sourceText, variable, e);
}
}
final boolean needsScores = expr.getSortField(bindings, false).needsScores();
return new ExpressionSearchScript(compiledScript, bindings, specialValue, needsScores);
}

55
modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
View File

@ -19,23 +19,21 @@
package org.elasticsearch.script.expression;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import org.apache.lucene.expressions.Bindings;
import org.apache.lucene.expressions.Expression;
import org.apache.lucene.expressions.SimpleBindings;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.Scorer;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.script.LeafSearchScript;
import org.elasticsearch.script.GeneralScriptException;
import org.elasticsearch.script.LeafSearchScript;
import org.elasticsearch.script.SearchScript;
import java.io.IOException;
import java.util.Map;
/**
* A bridge to evaluate an {@link Expression} against {@link Bindings} in the context
* of a {@link SearchScript}.
@ -44,16 +42,16 @@ class ExpressionSearchScript implements SearchScript {
final CompiledScript compiledScript;
final SimpleBindings bindings;
final ValueSource source;
final ReplaceableConstValueSource specialValue; // _value
final DoubleValuesSource source;
final ReplaceableConstDoubleValueSource specialValue; // _value
final boolean needsScores;
Scorer scorer;
int docid;
ExpressionSearchScript(CompiledScript c, SimpleBindings b, ReplaceableConstValueSource v, boolean needsScores) {
ExpressionSearchScript(CompiledScript c, SimpleBindings b, ReplaceableConstDoubleValueSource v, boolean needsScores) {
compiledScript = c;
bindings = b;
source = ((Expression)compiledScript.compiled()).getValueSource(bindings);
source = ((Expression)compiledScript.compiled()).getDoubleValuesSource(bindings);
specialValue = v;
this.needsScores = needsScores;
}
@ -63,15 +61,25 @@ class ExpressionSearchScript implements SearchScript {
return needsScores;
}
@Override
public LeafSearchScript getLeafSearchScript(final LeafReaderContext leaf) throws IOException {
return new LeafSearchScript() {
// Fake the scorer until setScorer is called.
DoubleValues values = source.getValues(leaf, new DoubleValues() {
@Override
public double doubleValue() throws IOException {
return Double.NaN;
}
FunctionValues values = source.getValues(Collections.singletonMap("scorer", Lucene.illegalScorer("Scores are not available in the current context")), leaf);
@Override
public boolean advanceExact(int doc) throws IOException {
return true;
}
});
double evaluate() {
try {
return values.doubleVal(docid);
return values.doubleValue();
} catch (Exception exception) {
throw new GeneralScriptException("Error evaluating " + compiledScript, exception);
}
@ -89,6 +97,11 @@ class ExpressionSearchScript implements SearchScript {
@Override
public void setDocument(int d) {
docid = d;
try {
values.advanceExact(d);
} catch (IOException e) {
throw new IllegalStateException("Can't advance to doc using " + compiledScript, e);
}
}
@Override
@ -96,7 +109,17 @@ class ExpressionSearchScript implements SearchScript {
scorer = s;
try {
// We have a new binding for the scorer so we need to reset the values
values = source.getValues(Collections.singletonMap("scorer", scorer), leaf);
values = source.getValues(leaf, new DoubleValues() {
@Override
public double doubleValue() throws IOException {
return scorer.score();
}
@Override
public boolean advanceExact(int doc) throws IOException {
return true;
}
});
} catch (IOException e) {
throw new IllegalStateException("Can't get values using " + compiledScript, e);
}

32
modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java → modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java
View File

@ -19,29 +19,32 @@
package org.elasticsearch.script.expression;
import java.io.IOException;
import java.util.Map;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.search.DoubleValuesSource;
import java.io.IOException;
/**
* A {@link ValueSource} which has a stub {@link FunctionValues} that holds a dynamically replaceable constant double.
* A {@link DoubleValuesSource} which has a stub {@link DoubleValues} that holds a dynamically replaceable constant double.
*/
final class ReplaceableConstValueSource extends ValueSource {
final ReplaceableConstFunctionValues fv;
final class ReplaceableConstDoubleValueSource extends DoubleValuesSource {
final ReplaceableConstDoubleValues fv;
ReplaceableConstValueSource() {
fv = new ReplaceableConstFunctionValues();
ReplaceableConstDoubleValueSource() {
fv = new ReplaceableConstDoubleValues();
}
@Override
@SuppressWarnings("rawtypes") // ValueSource uses a rawtype
public FunctionValues getValues(Map map, LeafReaderContext atomicReaderContext) throws IOException {
public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
return fv;
}
@Override
public boolean needsScores() {
return false;
}
@Override
public boolean equals(Object o) {
return o == this;
@ -52,11 +55,6 @@ final class ReplaceableConstValueSource extends ValueSource {
return System.identityHashCode(this);
}
@Override
public String description() {
return "replaceableConstDouble";
}
public void setValue(double v) {
fv.setValue(v);
}

14
modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java → modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java
View File

@ -19,13 +19,15 @@
package org.elasticsearch.script.expression;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.search.DoubleValues;
import java.io.IOException;
/**
* A support class for an executable expression script that allows the double returned
* by a {@link FunctionValues} to be modified.
* by a {@link DoubleValues} to be modified.
*/
final class ReplaceableConstFunctionValues extends FunctionValues {
final class ReplaceableConstDoubleValues extends DoubleValues {
private double value = 0;
void setValue(double value) {
@ -33,12 +35,12 @@ final class ReplaceableConstFunctionValues extends FunctionValues {
}
@Override
public double doubleVal(int doc) {
public double doubleValue() throws IOException {
return value;
}
@Override
public String toString(int i) {
return "ReplaceableConstFunctionValues: " + value;
public boolean advanceExact(int doc) throws IOException {
return true;
}
}

2
modules/lang-expression/src/main/plugin-metadata/plugin-security.policy
View File

@ -24,7 +24,7 @@ grant {
// expression runtime
permission org.elasticsearch.script.ClassPermission "java.lang.String";
permission org.elasticsearch.script.ClassPermission "org.apache.lucene.expressions.Expression";
permission org.elasticsearch.script.ClassPermission "org.apache.lucene.queries.function.FunctionValues";
permission org.elasticsearch.script.ClassPermission "org.apache.lucene.search.DoubleValues";
// available functions
permission org.elasticsearch.script.ClassPermission "java.lang.Math";
permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.MathUtil";

1
plugins/analysis-icu/licenses/lucene-analyzers-icu-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
8e6047ca31d0284e1ccac1ac2d5cbf1b8e3e1b04

1
plugins/analysis-icu/licenses/lucene-analyzers-icu-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
0242cfe1a026ec3afd5a567237f634da1d95fa24

1
plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
dca59de9397d9bd33ad0714cd9896fc1bb8f13ef

1
plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
fdefb46dde0019b8173d19fe4b164dca9efcba14

1
plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
0459b8e596e91ed00d5b36bc61adad53372c6491

1
plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
e75472bdbd3480ceddc570e747c70640429598d6

1
plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
4ad39a97e64f0a477a58934318b7f129f8c33c55

1
plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
31ebae985b936c2dcbe16126492deccbb5192ce8

1
plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
1ae21b511636da5abd5f498d20fb087fa07fc7c2

1
plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
684bc70e964bfb93224962b24912b1bab2a5b673

1
plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.4.0-snapshot-084f7a0.jar.sha1
View File

@ -1 +0,0 @@
9be4966458f88699fb09fb0f6a3a71017e7678e7

1
plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
07a2471f5da4a69be367851505338b53ceca0ef1