Scripting: Allow executable expression scripts for aggregations

Added several classes to support expressions being used for numerical
calculations in aggregations.  Expressions will still not compile
when used with mapping and update script contexts.

Closes #11596
Closes #11689
This commit is contained in:
Jack Conradson 2015-06-12 16:51:38 -07:00
parent 3fad16ac46
commit 917aeb7278
12 changed files with 367 additions and 54 deletions

View File

@ -19,7 +19,6 @@
package org.elasticsearch.index.mapper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import org.apache.lucene.document.Field;

View File

@ -56,6 +56,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.query.TemplateQueryParser;
import org.elasticsearch.script.expression.ExpressionScriptEngineService;
import org.elasticsearch.script.groovy.GroovyScriptEngineService;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.search.lookup.SearchLookup;
@ -232,6 +233,16 @@ public class ScriptService extends AbstractComponent implements Closeable {
if (canExecuteScript(lang, scriptEngineService, script.getType(), scriptContext) == false) {
throw new ScriptException("scripts of type [" + script.getType() + "], operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are disabled");
}
// special exception to prevent expressions from compiling as update or mapping scripts
boolean expression = scriptEngineService instanceof ExpressionScriptEngineService;
boolean notSupported = scriptContext.getKey().equals(ScriptContext.Standard.UPDATE.getKey()) ||
scriptContext.getKey().equals(ScriptContext.Standard.MAPPING.getKey());
if (expression && notSupported) {
throw new ScriptException("scripts of type [" + script.getType() + "]," +
" operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are not supported");
}
return compileInternal(script);
}

View File

@ -0,0 +1,89 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.expression;
import org.apache.lucene.expressions.Expression;
import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.script.ScriptException;
import java.util.HashMap;
import java.util.Map;
/**
* A bridge to evaluate an {@link Expression} against a map of variables in the context
* of an {@link ExecutableScript}.
*/
public class ExpressionExecutableScript implements ExecutableScript {
private final int NO_DOCUMENT = -1;
public final Expression expression;
public final Map<String, ReplaceableConstFunctionValues> functionValuesMap;
public final ReplaceableConstFunctionValues[] functionValuesArray;
public ExpressionExecutableScript(Object compiledScript, Map<String, Object> vars) {
expression = (Expression)compiledScript;
int functionValuesLength = expression.variables.length;
if (vars.size() != functionValuesLength) {
throw new ScriptException("The number of variables in an executable expression script [" +
functionValuesLength + "] must match the number of variables in the variable map" +
" [" + vars.size() + "].");
}
functionValuesArray = new ReplaceableConstFunctionValues[functionValuesLength];
functionValuesMap = new HashMap<>();
for (int functionValuesIndex = 0; functionValuesIndex < functionValuesLength; ++functionValuesIndex) {
String variableName = expression.variables[functionValuesIndex];
functionValuesArray[functionValuesIndex] = new ReplaceableConstFunctionValues();
functionValuesMap.put(variableName, functionValuesArray[functionValuesIndex]);
}
for (String varsName : vars.keySet()) {
setNextVar(varsName, vars.get(varsName));
}
}
@Override
public void setNextVar(String name, Object value) {
if (functionValuesMap.containsKey(name)) {
if (value instanceof Number) {
double doubleValue = ((Number)value).doubleValue();
functionValuesMap.get(name).setValue(doubleValue);
} else {
throw new ScriptException("Executable expressions scripts can only process numbers." +
" The variable [" + name + "] is not a number.");
}
} else {
throw new ScriptException("The variable [" + name + "] does not exist in the executable expressions script.");
}
}
@Override
public Object run() {
return expression.evaluate(NO_DOCUMENT, functionValuesArray);
}
@Override
public Object unwrap(Object value) {
return value;
}
}

View File

@ -31,7 +31,6 @@ import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.core.DateFieldMapper;
@ -172,7 +171,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
}
}
return new ExpressionScript((Expression)compiledScript, bindings, specialValue);
return new ExpressionSearchScript((Expression)compiledScript, bindings, specialValue);
}
protected ValueSource getMethodValueSource(MappedFieldType fieldType, IndexFieldData<?> fieldData, String fieldName, String methodName) {
@ -215,13 +214,14 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
}
@Override
public ExecutableScript executable(Object compiledScript, @Nullable Map<String, Object> vars) {
throw new UnsupportedOperationException("Cannot use expressions for updates");
public ExecutableScript executable(Object compiledScript, Map<String, Object> vars) {
return new ExpressionExecutableScript(compiledScript, vars);
}
@Override
public Object execute(Object compiledScript, Map<String, Object> vars) {
throw new UnsupportedOperationException("Cannot use expressions for updates");
ExpressionExecutableScript expressionExecutableScript = new ExpressionExecutableScript(compiledScript, vars);
return expressionExecutableScript.run();
}
@Override

View File

@ -38,7 +38,7 @@ import java.util.Map;
* A bridge to evaluate an {@link Expression} against {@link Bindings} in the context
* of a {@link SearchScript}.
*/
class ExpressionScript implements SearchScript {
class ExpressionSearchScript implements SearchScript {
final Expression expression;
final SimpleBindings bindings;
@ -47,7 +47,7 @@ class ExpressionScript implements SearchScript {
Scorer scorer;
int docid;
ExpressionScript(Expression e, SimpleBindings b, ReplaceableConstValueSource v) {
ExpressionSearchScript(Expression e, SimpleBindings b, ReplaceableConstValueSource v) {
expression = e;
bindings = b;
source = expression.getValueSource(bindings);

View File

@ -0,0 +1,44 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.expression;
import org.apache.lucene.queries.function.FunctionValues;
/**
* A support class for an executable expression script that allows the double returned
* by a {@link FunctionValues} to be modified.
*/
public class ReplaceableConstFunctionValues extends FunctionValues {
private double value = 0;
public void setValue(double value) {
this.value = value;
}
@Override
public double doubleVal(int doc) {
return value;
}
@Override
public String toString(int i) {
return "ReplaceableConstFunctionValues: " + value;
}
}

View File

@ -22,7 +22,6 @@ package org.elasticsearch.script.expression;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
import java.io.IOException;
import java.util.Map;
@ -31,16 +30,10 @@ import java.util.Map;
* A {@link ValueSource} which has a stub {@link FunctionValues} that holds a dynamically replaceable constant double.
*/
class ReplaceableConstValueSource extends ValueSource {
double value;
final FunctionValues fv;
final ReplaceableConstFunctionValues fv;
public ReplaceableConstValueSource() {
fv = new DoubleDocValues(this) {
@Override
public double doubleVal(int i) {
return value;
}
};
fv = new ReplaceableConstFunctionValues();
}
@Override
@ -64,6 +57,6 @@ class ReplaceableConstValueSource extends ValueSource {
}
public void setValue(double v) {
value = v;
fv.setValue(v);
}
}

View File

@ -72,7 +72,7 @@ public class CustomScriptContextTests extends ElasticsearchIntegrationTest {
}
CompiledScript compiledScript = scriptService.compile(new Script("1", ScriptService.ScriptType.INLINE, "expression", null),
randomFrom(ScriptContext.Standard.values()));
randomFrom(new ScriptContext[] {ScriptContext.Standard.AGGS, ScriptContext.Standard.SEARCH}));
assertThat(compiledScript, notNullValue());
compiledScript = scriptService.compile(new Script("1", ScriptService.ScriptType.INLINE, "mustache", null),

View File

@ -162,7 +162,7 @@ public class IndexedScriptTests extends ElasticsearchIntegrationTest {
fail("update script should have been rejected");
} catch(Exception e) {
assertThat(e.getMessage(), containsString("failed to execute script"));
assertThat(e.getCause().toString(), containsString("scripts of type [indexed], operation [update] and lang [expression] are disabled"));
assertThat(e.getCause().getMessage(), containsString("scripts of type [indexed], operation [update] and lang [expression] are disabled"));
}
try {
String query = "{ \"script_fields\" : { \"test1\" : { \"script_id\" : \"script1\", \"lang\":\"expression\" }}}";

View File

@ -142,8 +142,7 @@ public class OnDiskScriptTests extends ElasticsearchIntegrationTest {
fail("update script should have been rejected");
} catch (Exception e) {
assertThat(e.getMessage(), containsString("failed to execute script"));
assertThat(e.getCause().toString(),
containsString("scripts of type [file], operation [update] and lang [mustache] are disabled"));
assertThat(e.getCause().getMessage(), containsString("scripts of type [file], operation [update] and lang [mustache] are disabled"));
}
}

View File

@ -159,8 +159,8 @@ public class ScriptServiceTests extends ElasticsearchTestCase {
CompiledScript groovyScript = scriptService.compile(
new Script("file_script", ScriptType.FILE, GroovyScriptEngineService.NAME, null), randomFrom(scriptContexts));
assertThat(groovyScript.lang(), equalTo(GroovyScriptEngineService.NAME));
CompiledScript expressionScript = scriptService.compile(new Script("file_script", ScriptType.FILE,
ExpressionScriptEngineService.NAME, null), randomFrom(scriptContexts));
CompiledScript expressionScript = scriptService.compile(new Script("file_script", ScriptType.FILE, ExpressionScriptEngineService.NAME,
null), randomFrom(new ScriptContext[] {ScriptContext.Standard.AGGS, ScriptContext.Standard.SEARCH}));
assertThat(expressionScript.lang(), equalTo(ExpressionScriptEngineService.NAME));
}
@ -207,9 +207,12 @@ public class ScriptServiceTests extends ElasticsearchTestCase {
assertCompileRejected(GroovyScriptEngineService.NAME, "script", ScriptType.INDEXED, scriptContext);
assertCompileAccepted(GroovyScriptEngineService.NAME, "file_script", ScriptType.FILE, scriptContext);
//expression engine is sandboxed, all scripts are enabled by default
if (!scriptContext.getKey().equals(ScriptContext.Standard.MAPPING.getKey()) &&
!scriptContext.getKey().equals(ScriptContext.Standard.UPDATE.getKey())) {
assertCompileAccepted(ExpressionScriptEngineService.NAME, "script", ScriptType.INLINE, scriptContext);
assertCompileAccepted(ExpressionScriptEngineService.NAME, "script", ScriptType.INDEXED, scriptContext);
assertCompileAccepted(ExpressionScriptEngineService.NAME, "file_script", ScriptType.FILE, scriptContext);
}
//mustache engine is sandboxed, all scripts are enabled by default
assertCompileAccepted(MustacheScriptEngineService.NAME, "script", ScriptType.INLINE, scriptContext);
assertCompileAccepted(MustacheScriptEngineService.NAME, "script", ScriptType.INDEXED, scriptContext);
@ -311,6 +314,12 @@ public class ScriptServiceTests extends ElasticsearchTestCase {
//Otherwise they are always considered file ones as they can be found in the static cache.
String script = scriptType == ScriptType.FILE ? "file_script" : "script";
for (ScriptContext scriptContext : this.scriptContexts) {
// skip script contexts that aren't allowed for expressions
if (scriptEngineService instanceof ExpressionScriptEngineService &&
(scriptContext.getKey().equals(ScriptContext.Standard.MAPPING.getKey()) ||
scriptContext.getKey().equals(ScriptContext.Standard.UPDATE.getKey()))) {
continue;
}
//fallback mechanism: 1) engine specific settings 2) op based settings 3) source based settings
ScriptMode scriptMode = engineSettings.get(scriptEngineService.types()[0] + "." + scriptType + "." + scriptContext.getKey());
if (scriptMode == null) {

View File

@ -19,29 +19,45 @@
package org.elasticsearch.script.expression;
import org.apache.lucene.expressions.Expression;
import org.apache.lucene.expressions.js.JavascriptCompiler;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.search.SearchPhaseExecutionException;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.action.update.UpdateRequestBuilder;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptException;
import org.elasticsearch.script.ScriptService.ScriptType;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram.Bucket;
import org.elasticsearch.search.aggregations.metrics.stats.Stats;
import org.elasticsearch.search.aggregations.pipeline.SimpleValue;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.seriesArithmetic;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.notNullValue;
public class ExpressionScriptTests extends ElasticsearchIntegrationTest {
@ -49,7 +65,7 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest {
ensureGreen("test");
Map<String, Object> paramsMap = new HashMap<>();
assert(params.length % 2 == 0);
assert (params.length % 2 == 0);
for (int i = 0; i < params.length; i += 2) {
paramsMap.put(params[i].toString(), params[i + 1]);
}
@ -57,7 +73,7 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest {
SearchRequestBuilder req = client().prepareSearch().setIndices("test");
req.setQuery(QueryBuilders.matchAllQuery())
.addSort(SortBuilders.fieldSort("_uid")
.order(SortOrder.ASC))
.order(SortOrder.ASC))
.addScriptField("foo", new Script(script, ScriptType.INLINE, "expression", paramsMap));
return req;
}
@ -370,7 +386,7 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest {
SearchRequestBuilder req = client().prepareSearch().setIndices("test");
req.setQuery(QueryBuilders.matchAllQuery())
.addAggregation(
.addAggregation(
AggregationBuilders.terms("term_agg").field("text")
.script(new Script("_value", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null)));
@ -389,4 +405,157 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest {
assertThat(message + "should have contained text variable error",
message.contains("text variable"), equalTo(true));
}
// series of unit test for using expressions as executable scripts
public void testExecutableScripts() throws Exception {
Map<String, Object> vars = new HashMap<>();
vars.put("a", 2.5);
vars.put("b", 3);
vars.put("xyz", -1);
Expression expr = JavascriptCompiler.compile("a+b+xyz");
ExpressionExecutableScript ees = new ExpressionExecutableScript(expr, vars);
assertEquals((Double) ees.run(), 4.5, 0.001);
ees.setNextVar("b", -2.5);
assertEquals((Double) ees.run(), -1, 0.001);
ees.setNextVar("a", -2.5);
ees.setNextVar("b", -2.5);
ees.setNextVar("xyz", -2.5);
assertEquals((Double) ees.run(), -7.5, 0.001);
String message;
try {
vars = new HashMap<>();
vars.put("a", 1);
ees = new ExpressionExecutableScript(expr, vars);
ees.run();
fail("An incorrect number of variables were allowed to be used in an expression.");
} catch (ScriptException se) {
message = se.getMessage();
assertThat(message + " should have contained number of variables", message.contains("number of variables"), equalTo(true));
}
try {
vars = new HashMap<>();
vars.put("a", 1);
vars.put("b", 3);
vars.put("c", -1);
ees = new ExpressionExecutableScript(expr, vars);
ees.run();
fail("A variable was allowed to be set that does not exist in the expression.");
} catch (ScriptException se) {
message = se.getMessage();
assertThat(message + " should have contained does not exist in", message.contains("does not exist in"), equalTo(true));
}
try {
vars = new HashMap<>();
vars.put("a", 1);
vars.put("b", 3);
vars.put("xyz", "hello");
ees = new ExpressionExecutableScript(expr, vars);
ees.run();
fail("A non-number was allowed to be use in the expression.");
} catch (ScriptException se) {
message = se.getMessage();
assertThat(message + " should have contained process numbers", message.contains("process numbers"), equalTo(true));
}
}
// test to make sure expressions are not allowed to be used as update scripts
public void testInvalidUpdateScript() throws Exception {
try {
createIndex("test_index");
ensureGreen("test_index");
indexRandom(true, client().prepareIndex("test_index", "doc", "1").setSource("text_field", "text"));
UpdateRequestBuilder urb = client().prepareUpdate().setIndex("test_index");
urb.setType("doc");
urb.setId("1");
urb.setScript(new Script("0", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null));
urb.get();
fail("Expression scripts should not be allowed to run as update scripts.");
} catch (Exception e) {
String message = e.getMessage();
assertThat(message + " should have contained failed to execute", message.contains("failed to execute"), equalTo(true));
message = e.getCause().getMessage();
assertThat(message + " should have contained not supported", message.contains("not supported"), equalTo(true));
}
}
// test to make sure expressions are not allowed to be used as mapping scripts
public void testInvalidMappingScript() throws Exception{
try {
createIndex("test_index");
ensureGreen("test_index");
XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
builder.startObject("transform");
builder.field("script", "1.0");
builder.field("lang", ExpressionScriptEngineService.NAME);
builder.endObject();
builder.startObject("properties");
builder.startObject("double_field");
builder.field("type", "double");
builder.endObject();
builder.endObject();
builder.endObject();
client().admin().indices().preparePutMapping("test_index").setType("trans_test").setSource(builder).get();
client().prepareIndex("test_index", "trans_test", "1").setSource("double_field", 0.0).get();
fail("Expression scripts should not be allowed to run as mapping scripts.");
} catch (Exception e) {
String message = ExceptionsHelper.detailedMessage(e);
assertThat(message + " should have contained failed to parse", message.contains("failed to parse"), equalTo(true));
assertThat(message + " should have contained not supported", message.contains("not supported"), equalTo(true));
}
}
// test to make sure expressions are allowed to be used for reduce in pipeline aggregations
public void testPipelineAggregationScript() throws Exception {
createIndex("agg_index");
ensureGreen("agg_index");
indexRandom(true,
client().prepareIndex("agg_index", "doc", "1").setSource("one", 1.0, "two", 2.0, "three", 3.0, "four", 4.0),
client().prepareIndex("agg_index", "doc", "2").setSource("one", 2.0, "two", 2.0, "three", 3.0, "four", 4.0),
client().prepareIndex("agg_index", "doc", "3").setSource("one", 3.0, "two", 2.0, "three", 3.0, "four", 4.0),
client().prepareIndex("agg_index", "doc", "4").setSource("one", 4.0, "two", 2.0, "three", 3.0, "four", 4.0),
client().prepareIndex("agg_index", "doc", "5").setSource("one", 5.0, "two", 2.0, "three", 3.0, "four", 4.0));
SearchResponse response = client()
.prepareSearch("agg_index")
.addAggregation(
histogram("histogram")
.field("one")
.interval(2)
.subAggregation(sum("twoSum").field("two"))
.subAggregation(sum("threeSum").field("three"))
.subAggregation(sum("fourSum").field("four"))
.subAggregation(
seriesArithmetic("totalSum").setBucketsPaths("twoSum", "threeSum", "fourSum").script(
new Script("_value0 + _value1 + _value2", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null)))).execute().actionGet();
InternalHistogram<Bucket> histogram = response.getAggregations().get("histogram");
assertThat(histogram, notNullValue());
assertThat(histogram.getName(), equalTo("histogram"));
List<Bucket> buckets = histogram.getBuckets();
for (int bucketCount = 0; bucketCount < buckets.size(); ++bucketCount) {
Histogram.Bucket bucket = buckets.get(bucketCount);
if (bucket.getDocCount() == 1) {
SimpleValue seriesArithmetic = bucket.getAggregations().get("totalSum");
assertThat(seriesArithmetic, notNullValue());
double seriesArithmeticValue = seriesArithmetic.value();
assertEquals(9.0, seriesArithmeticValue, 0.001);
} else if (bucket.getDocCount() == 2) {
SimpleValue seriesArithmetic = bucket.getAggregations().get("totalSum");
assertThat(seriesArithmetic, notNullValue());
double seriesArithmeticValue = seriesArithmetic.value();
assertEquals(18.0, seriesArithmeticValue, 0.001);
} else {
fail("Incorrect number of documents in a bucket in the histogram.");
}
}
}
}