From 917aeb7278c6065000d0a3a11bda7021e9bf2d85 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Fri, 12 Jun 2015 16:51:38 -0700 Subject: [PATCH] Scripting: Allow executable expression scripts for aggregations Added several classes to support expressions being used for numerical calculations in aggregations. Expressions will still not compile when used with mapping and update script contexts. Closes #11596 Closes #11689 --- .../index/mapper/DocumentParser.java | 1 - .../elasticsearch/script/ScriptService.java | 11 + .../ExpressionExecutableScript.java | 89 +++++++ .../ExpressionScriptEngineService.java | 10 +- ...cript.java => ExpressionSearchScript.java} | 4 +- .../ReplaceableConstFunctionValues.java | 44 ++++ .../ReplaceableConstValueSource.java | 13 +- .../script/CustomScriptContextTests.java | 2 +- .../script/IndexedScriptTests.java | 2 +- .../script/OnDiskScriptTests.java | 3 +- .../script/ScriptServiceTests.java | 19 +- .../expression/ExpressionScriptTests.java | 223 +++++++++++++++--- 12 files changed, 367 insertions(+), 54 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java rename core/src/main/java/org/elasticsearch/script/expression/{ExpressionScript.java => ExpressionSearchScript.java} (96%) create mode 100644 core/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 344e65097ad..79ca3982442 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import org.apache.lucene.document.Field; diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java index 60da06ecf93..9ca72b29c78 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java @@ -56,6 +56,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.index.query.TemplateQueryParser; +import org.elasticsearch.script.expression.ExpressionScriptEngineService; import org.elasticsearch.script.groovy.GroovyScriptEngineService; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.SearchLookup; @@ -232,6 +233,16 @@ public class ScriptService extends AbstractComponent implements Closeable { if (canExecuteScript(lang, scriptEngineService, script.getType(), scriptContext) == false) { throw new ScriptException("scripts of type [" + script.getType() + "], operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are disabled"); } + + // special exception to prevent expressions from compiling as update or mapping scripts + boolean expression = scriptEngineService instanceof ExpressionScriptEngineService; + boolean notSupported = scriptContext.getKey().equals(ScriptContext.Standard.UPDATE.getKey()) || + scriptContext.getKey().equals(ScriptContext.Standard.MAPPING.getKey()); + if (expression && notSupported) { + throw new ScriptException("scripts of type [" + script.getType() + "]," + + " operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are not supported"); + } + return compileInternal(script); } diff --git a/core/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java b/core/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java new file mode 100644 index 00000000000..ff41fb8fd78 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java @@ -0,0 +1,89 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.expression; + +import org.apache.lucene.expressions.Expression; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.ScriptException; + +import java.util.HashMap; +import java.util.Map; + +/** + * A bridge to evaluate an {@link Expression} against a map of variables in the context + * of an {@link ExecutableScript}. + */ +public class ExpressionExecutableScript implements ExecutableScript { + + private final int NO_DOCUMENT = -1; + + public final Expression expression; + public final Map functionValuesMap; + public final ReplaceableConstFunctionValues[] functionValuesArray; + + public ExpressionExecutableScript(Object compiledScript, Map vars) { + expression = (Expression)compiledScript; + int functionValuesLength = expression.variables.length; + + if (vars.size() != functionValuesLength) { + throw new ScriptException("The number of variables in an executable expression script [" + + functionValuesLength + "] must match the number of variables in the variable map" + + " [" + vars.size() + "]."); + } + + functionValuesArray = new ReplaceableConstFunctionValues[functionValuesLength]; + functionValuesMap = new HashMap<>(); + + for (int functionValuesIndex = 0; functionValuesIndex < functionValuesLength; ++functionValuesIndex) { + String variableName = expression.variables[functionValuesIndex]; + functionValuesArray[functionValuesIndex] = new ReplaceableConstFunctionValues(); + functionValuesMap.put(variableName, functionValuesArray[functionValuesIndex]); + } + + for (String varsName : vars.keySet()) { + setNextVar(varsName, vars.get(varsName)); + } + } + + @Override + public void setNextVar(String name, Object value) { + if (functionValuesMap.containsKey(name)) { + if (value instanceof Number) { + double doubleValue = ((Number)value).doubleValue(); + functionValuesMap.get(name).setValue(doubleValue); + } else { + throw new ScriptException("Executable expressions scripts can only process numbers." + + " The variable [" + name + "] is not a number."); + } + } else { + throw new ScriptException("The variable [" + name + "] does not exist in the executable expressions script."); + } + } + + @Override + public Object run() { + return expression.evaluate(NO_DOCUMENT, functionValuesArray); + } + + @Override + public Object unwrap(Object value) { + return value; + } +} diff --git a/core/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/core/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java index 26d89a0ccaf..2c4c8f4a25b 100644 --- a/core/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java +++ b/core/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.core.DateFieldMapper; @@ -172,7 +171,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements } } - return new ExpressionScript((Expression)compiledScript, bindings, specialValue); + return new ExpressionSearchScript((Expression)compiledScript, bindings, specialValue); } protected ValueSource getMethodValueSource(MappedFieldType fieldType, IndexFieldData fieldData, String fieldName, String methodName) { @@ -215,13 +214,14 @@ public class ExpressionScriptEngineService extends AbstractComponent implements } @Override - public ExecutableScript executable(Object compiledScript, @Nullable Map vars) { - throw new UnsupportedOperationException("Cannot use expressions for updates"); + public ExecutableScript executable(Object compiledScript, Map vars) { + return new ExpressionExecutableScript(compiledScript, vars); } @Override public Object execute(Object compiledScript, Map vars) { - throw new UnsupportedOperationException("Cannot use expressions for updates"); + ExpressionExecutableScript expressionExecutableScript = new ExpressionExecutableScript(compiledScript, vars); + return expressionExecutableScript.run(); } @Override diff --git a/core/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java b/core/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java similarity index 96% rename from core/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java rename to core/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java index 9f4067a5997..59a73a3ff6d 100644 --- a/core/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java +++ b/core/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java @@ -38,7 +38,7 @@ import java.util.Map; * A bridge to evaluate an {@link Expression} against {@link Bindings} in the context * of a {@link SearchScript}. */ -class ExpressionScript implements SearchScript { +class ExpressionSearchScript implements SearchScript { final Expression expression; final SimpleBindings bindings; @@ -47,7 +47,7 @@ class ExpressionScript implements SearchScript { Scorer scorer; int docid; - ExpressionScript(Expression e, SimpleBindings b, ReplaceableConstValueSource v) { + ExpressionSearchScript(Expression e, SimpleBindings b, ReplaceableConstValueSource v) { expression = e; bindings = b; source = expression.getValueSource(bindings); diff --git a/core/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java b/core/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java new file mode 100644 index 00000000000..b3ebfab66ed --- /dev/null +++ b/core/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java @@ -0,0 +1,44 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.expression; + +import org.apache.lucene.queries.function.FunctionValues; + +/** + * A support class for an executable expression script that allows the double returned + * by a {@link FunctionValues} to be modified. + */ +public class ReplaceableConstFunctionValues extends FunctionValues { + private double value = 0; + + public void setValue(double value) { + this.value = value; + } + + @Override + public double doubleVal(int doc) { + return value; + } + + @Override + public String toString(int i) { + return "ReplaceableConstFunctionValues: " + value; + } +} diff --git a/core/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java b/core/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java index 95eb5e4ab1d..c9bcc239f28 100644 --- a/core/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java +++ b/core/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java @@ -22,7 +22,6 @@ package org.elasticsearch.script.expression; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.queries.function.docvalues.DoubleDocValues; import java.io.IOException; import java.util.Map; @@ -31,16 +30,10 @@ import java.util.Map; * A {@link ValueSource} which has a stub {@link FunctionValues} that holds a dynamically replaceable constant double. */ class ReplaceableConstValueSource extends ValueSource { - double value; - final FunctionValues fv; + final ReplaceableConstFunctionValues fv; public ReplaceableConstValueSource() { - fv = new DoubleDocValues(this) { - @Override - public double doubleVal(int i) { - return value; - } - }; + fv = new ReplaceableConstFunctionValues(); } @Override @@ -64,6 +57,6 @@ class ReplaceableConstValueSource extends ValueSource { } public void setValue(double v) { - value = v; + fv.setValue(v); } } diff --git a/core/src/test/java/org/elasticsearch/script/CustomScriptContextTests.java b/core/src/test/java/org/elasticsearch/script/CustomScriptContextTests.java index c38aec157ff..16006429baa 100644 --- a/core/src/test/java/org/elasticsearch/script/CustomScriptContextTests.java +++ b/core/src/test/java/org/elasticsearch/script/CustomScriptContextTests.java @@ -72,7 +72,7 @@ public class CustomScriptContextTests extends ElasticsearchIntegrationTest { } CompiledScript compiledScript = scriptService.compile(new Script("1", ScriptService.ScriptType.INLINE, "expression", null), - randomFrom(ScriptContext.Standard.values())); + randomFrom(new ScriptContext[] {ScriptContext.Standard.AGGS, ScriptContext.Standard.SEARCH})); assertThat(compiledScript, notNullValue()); compiledScript = scriptService.compile(new Script("1", ScriptService.ScriptType.INLINE, "mustache", null), diff --git a/core/src/test/java/org/elasticsearch/script/IndexedScriptTests.java b/core/src/test/java/org/elasticsearch/script/IndexedScriptTests.java index c88f1014094..f4696d8f8e3 100644 --- a/core/src/test/java/org/elasticsearch/script/IndexedScriptTests.java +++ b/core/src/test/java/org/elasticsearch/script/IndexedScriptTests.java @@ -162,7 +162,7 @@ public class IndexedScriptTests extends ElasticsearchIntegrationTest { fail("update script should have been rejected"); } catch(Exception e) { assertThat(e.getMessage(), containsString("failed to execute script")); - assertThat(e.getCause().toString(), containsString("scripts of type [indexed], operation [update] and lang [expression] are disabled")); + assertThat(e.getCause().getMessage(), containsString("scripts of type [indexed], operation [update] and lang [expression] are disabled")); } try { String query = "{ \"script_fields\" : { \"test1\" : { \"script_id\" : \"script1\", \"lang\":\"expression\" }}}"; diff --git a/core/src/test/java/org/elasticsearch/script/OnDiskScriptTests.java b/core/src/test/java/org/elasticsearch/script/OnDiskScriptTests.java index d10e8888e04..adc5351d57b 100644 --- a/core/src/test/java/org/elasticsearch/script/OnDiskScriptTests.java +++ b/core/src/test/java/org/elasticsearch/script/OnDiskScriptTests.java @@ -142,8 +142,7 @@ public class OnDiskScriptTests extends ElasticsearchIntegrationTest { fail("update script should have been rejected"); } catch (Exception e) { assertThat(e.getMessage(), containsString("failed to execute script")); - assertThat(e.getCause().toString(), - containsString("scripts of type [file], operation [update] and lang [mustache] are disabled")); + assertThat(e.getCause().getMessage(), containsString("scripts of type [file], operation [update] and lang [mustache] are disabled")); } } diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java index a43edcbc0b7..b2f7a186767 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java @@ -159,8 +159,8 @@ public class ScriptServiceTests extends ElasticsearchTestCase { CompiledScript groovyScript = scriptService.compile( new Script("file_script", ScriptType.FILE, GroovyScriptEngineService.NAME, null), randomFrom(scriptContexts)); assertThat(groovyScript.lang(), equalTo(GroovyScriptEngineService.NAME)); - CompiledScript expressionScript = scriptService.compile(new Script("file_script", ScriptType.FILE, - ExpressionScriptEngineService.NAME, null), randomFrom(scriptContexts)); + CompiledScript expressionScript = scriptService.compile(new Script("file_script", ScriptType.FILE, ExpressionScriptEngineService.NAME, + null), randomFrom(new ScriptContext[] {ScriptContext.Standard.AGGS, ScriptContext.Standard.SEARCH})); assertThat(expressionScript.lang(), equalTo(ExpressionScriptEngineService.NAME)); } @@ -207,9 +207,12 @@ public class ScriptServiceTests extends ElasticsearchTestCase { assertCompileRejected(GroovyScriptEngineService.NAME, "script", ScriptType.INDEXED, scriptContext); assertCompileAccepted(GroovyScriptEngineService.NAME, "file_script", ScriptType.FILE, scriptContext); //expression engine is sandboxed, all scripts are enabled by default - assertCompileAccepted(ExpressionScriptEngineService.NAME, "script", ScriptType.INLINE, scriptContext); - assertCompileAccepted(ExpressionScriptEngineService.NAME, "script", ScriptType.INDEXED, scriptContext); - assertCompileAccepted(ExpressionScriptEngineService.NAME, "file_script", ScriptType.FILE, scriptContext); + if (!scriptContext.getKey().equals(ScriptContext.Standard.MAPPING.getKey()) && + !scriptContext.getKey().equals(ScriptContext.Standard.UPDATE.getKey())) { + assertCompileAccepted(ExpressionScriptEngineService.NAME, "script", ScriptType.INLINE, scriptContext); + assertCompileAccepted(ExpressionScriptEngineService.NAME, "script", ScriptType.INDEXED, scriptContext); + assertCompileAccepted(ExpressionScriptEngineService.NAME, "file_script", ScriptType.FILE, scriptContext); + } //mustache engine is sandboxed, all scripts are enabled by default assertCompileAccepted(MustacheScriptEngineService.NAME, "script", ScriptType.INLINE, scriptContext); assertCompileAccepted(MustacheScriptEngineService.NAME, "script", ScriptType.INDEXED, scriptContext); @@ -311,6 +314,12 @@ public class ScriptServiceTests extends ElasticsearchTestCase { //Otherwise they are always considered file ones as they can be found in the static cache. String script = scriptType == ScriptType.FILE ? "file_script" : "script"; for (ScriptContext scriptContext : this.scriptContexts) { + // skip script contexts that aren't allowed for expressions + if (scriptEngineService instanceof ExpressionScriptEngineService && + (scriptContext.getKey().equals(ScriptContext.Standard.MAPPING.getKey()) || + scriptContext.getKey().equals(ScriptContext.Standard.UPDATE.getKey()))) { + continue; + } //fallback mechanism: 1) engine specific settings 2) op based settings 3) source based settings ScriptMode scriptMode = engineSettings.get(scriptEngineService.types()[0] + "." + scriptType + "." + scriptContext.getKey()); if (scriptMode == null) { diff --git a/core/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java b/core/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java index 065e03cb731..e8895a3fc05 100644 --- a/core/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java +++ b/core/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java @@ -19,29 +19,45 @@ package org.elasticsearch.script.expression; +import org.apache.lucene.expressions.Expression; +import org.apache.lucene.expressions.js.JavascriptCompiler; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.action.update.UpdateRequestBuilder; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptException; import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram; +import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram.Bucket; import org.elasticsearch.search.aggregations.metrics.stats.Stats; +import org.elasticsearch.search.aggregations.pipeline.SimpleValue; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import java.util.HashMap; +import java.util.List; import java.util.Map; +import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; +import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; +import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.seriesArithmetic; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.notNullValue; public class ExpressionScriptTests extends ElasticsearchIntegrationTest { @@ -49,15 +65,15 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest { ensureGreen("test"); Map paramsMap = new HashMap<>(); - assert(params.length % 2 == 0); + assert (params.length % 2 == 0); for (int i = 0; i < params.length; i += 2) { paramsMap.put(params[i].toString(), params[i + 1]); } SearchRequestBuilder req = client().prepareSearch().setIndices("test"); req.setQuery(QueryBuilders.matchAllQuery()) - .addSort(SortBuilders.fieldSort("_uid") -.order(SortOrder.ASC)) + .addSort(SortBuilders.fieldSort("_uid") + .order(SortOrder.ASC)) .addScriptField("foo", new Script(script, ScriptType.INLINE, "expression", paramsMap)); return req; } @@ -84,9 +100,9 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest { createIndex("test"); ensureGreen("test"); indexRandom(true, - client().prepareIndex("test", "doc", "1").setSource("text", "hello goodbye"), - client().prepareIndex("test", "doc", "2").setSource("text", "hello hello hello goodbye"), - client().prepareIndex("test", "doc", "3").setSource("text", "hello hello goodebye")); + client().prepareIndex("test", "doc", "1").setSource("text", "hello goodbye"), + client().prepareIndex("test", "doc", "2").setSource("text", "hello hello hello goodbye"), + client().prepareIndex("test", "doc", "3").setSource("text", "hello hello goodebye")); ScoreFunctionBuilder score = ScoreFunctionBuilders.scriptFunction(new Script("1 / _score", ScriptType.INLINE, "expression", null)); SearchRequestBuilder req = client().prepareSearch().setIndices("test"); req.setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("text", "hello"), score).boostMode("replace")); @@ -242,9 +258,9 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest { createIndex("test"); ensureGreen("test"); indexRandom(true, - client().prepareIndex("test", "doc", "1").setSource("x", 10), - client().prepareIndex("test", "doc", "2").setSource("x", 3), - client().prepareIndex("test", "doc", "3").setSource("x", 5)); + client().prepareIndex("test", "doc", "1").setSource("x", 10), + client().prepareIndex("test", "doc", "2").setSource("x", 3), + client().prepareIndex("test", "doc", "3").setSource("x", 5)); // a = int, b = double, c = long String script = "doc['x'] * a + b + ((c + doc['x']) > 5000000009 ? 1 : 0)"; SearchResponse rsp = buildRequest(script, "a", 2, "b", 3.5, "c", 5000000000L).get(); @@ -262,9 +278,9 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest { fail("Expected expression compilation failure"); } catch (SearchPhaseExecutionException e) { assertThat(e.toString() + "should have contained ExpressionScriptCompilationException", - e.toString().contains("ExpressionScriptCompilationException"), equalTo(true)); + e.toString().contains("ExpressionScriptCompilationException"), equalTo(true)); assertThat(e.toString() + "should have contained compilation failure", - e.toString().contains("Failed to parse expression"), equalTo(true)); + e.toString().contains("Failed to parse expression"), equalTo(true)); } } @@ -275,9 +291,9 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest { fail("Expected string parameter to cause failure"); } catch (SearchPhaseExecutionException e) { assertThat(e.toString() + "should have contained ExpressionScriptCompilationException", - e.toString().contains("ExpressionScriptCompilationException"), equalTo(true)); + e.toString().contains("ExpressionScriptCompilationException"), equalTo(true)); assertThat(e.toString() + "should have contained non-numeric parameter error", - e.toString().contains("must be a numeric type"), equalTo(true)); + e.toString().contains("must be a numeric type"), equalTo(true)); } } @@ -288,9 +304,9 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest { fail("Expected text field to cause execution failure"); } catch (SearchPhaseExecutionException e) { assertThat(e.toString() + "should have contained ExpressionScriptCompilationException", - e.toString().contains("ExpressionScriptCompilationException"), equalTo(true)); + e.toString().contains("ExpressionScriptCompilationException"), equalTo(true)); assertThat(e.toString() + "should have contained non-numeric field error", - e.toString().contains("must be numeric"), equalTo(true)); + e.toString().contains("must be numeric"), equalTo(true)); } } @@ -301,9 +317,9 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest { fail("Expected bogus variable to cause execution failure"); } catch (SearchPhaseExecutionException e) { assertThat(e.toString() + "should have contained ExpressionScriptCompilationException", - e.toString().contains("ExpressionScriptCompilationException"), equalTo(true)); + e.toString().contains("ExpressionScriptCompilationException"), equalTo(true)); assertThat(e.toString() + "should have contained unknown variable error", - e.toString().contains("Unknown variable"), equalTo(true)); + e.toString().contains("Unknown variable"), equalTo(true)); } } @@ -338,14 +354,14 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest { createIndex("test"); ensureGreen("test"); indexRandom(true, - client().prepareIndex("test", "doc", "1").setSource("x", 5, "y", 1.2), - client().prepareIndex("test", "doc", "2").setSource("x", 10, "y", 1.4), - client().prepareIndex("test", "doc", "3").setSource("x", 13, "y", 1.8)); + client().prepareIndex("test", "doc", "1").setSource("x", 5, "y", 1.2), + client().prepareIndex("test", "doc", "2").setSource("x", 10, "y", 1.4), + client().prepareIndex("test", "doc", "3").setSource("x", 13, "y", 1.8)); SearchRequestBuilder req = client().prepareSearch().setIndices("test"); req.setQuery(QueryBuilders.matchAllQuery()) - .addAggregation(AggregationBuilders.stats("int_agg").field("x").script("_value * 3").lang(ExpressionScriptEngineService.NAME)) - .addAggregation(AggregationBuilders.stats("double_agg").field("y").script("_value - 1.1").lang(ExpressionScriptEngineService.NAME)); + .addAggregation(AggregationBuilders.stats("int_agg").field("x").script("_value * 3").lang(ExpressionScriptEngineService.NAME)) + .addAggregation(AggregationBuilders.stats("double_agg").field("y").script("_value - 1.1").lang(ExpressionScriptEngineService.NAME)); SearchResponse rsp = req.get(); assertEquals(3, rsp.getHits().getTotalHits()); @@ -370,9 +386,9 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest { SearchRequestBuilder req = client().prepareSearch().setIndices("test"); req.setQuery(QueryBuilders.matchAllQuery()) -.addAggregation( - AggregationBuilders.terms("term_agg").field("text") - .script(new Script("_value", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null))); + .addAggregation( + AggregationBuilders.terms("term_agg").field("text") + .script(new Script("_value", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null))); String message; try { @@ -385,8 +401,161 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest { message = e.toString(); } assertThat(message + "should have contained ExpressionScriptExecutionException", - message.contains("ExpressionScriptExecutionException"), equalTo(true)); + message.contains("ExpressionScriptExecutionException"), equalTo(true)); assertThat(message + "should have contained text variable error", - message.contains("text variable"), equalTo(true)); + message.contains("text variable"), equalTo(true)); + } + + // series of unit test for using expressions as executable scripts + public void testExecutableScripts() throws Exception { + Map vars = new HashMap<>(); + vars.put("a", 2.5); + vars.put("b", 3); + vars.put("xyz", -1); + + Expression expr = JavascriptCompiler.compile("a+b+xyz"); + + ExpressionExecutableScript ees = new ExpressionExecutableScript(expr, vars); + assertEquals((Double) ees.run(), 4.5, 0.001); + + ees.setNextVar("b", -2.5); + assertEquals((Double) ees.run(), -1, 0.001); + + ees.setNextVar("a", -2.5); + ees.setNextVar("b", -2.5); + ees.setNextVar("xyz", -2.5); + assertEquals((Double) ees.run(), -7.5, 0.001); + + String message; + + try { + vars = new HashMap<>(); + vars.put("a", 1); + ees = new ExpressionExecutableScript(expr, vars); + ees.run(); + fail("An incorrect number of variables were allowed to be used in an expression."); + } catch (ScriptException se) { + message = se.getMessage(); + assertThat(message + " should have contained number of variables", message.contains("number of variables"), equalTo(true)); + } + + try { + vars = new HashMap<>(); + vars.put("a", 1); + vars.put("b", 3); + vars.put("c", -1); + ees = new ExpressionExecutableScript(expr, vars); + ees.run(); + fail("A variable was allowed to be set that does not exist in the expression."); + } catch (ScriptException se) { + message = se.getMessage(); + assertThat(message + " should have contained does not exist in", message.contains("does not exist in"), equalTo(true)); + } + + try { + vars = new HashMap<>(); + vars.put("a", 1); + vars.put("b", 3); + vars.put("xyz", "hello"); + ees = new ExpressionExecutableScript(expr, vars); + ees.run(); + fail("A non-number was allowed to be use in the expression."); + } catch (ScriptException se) { + message = se.getMessage(); + assertThat(message + " should have contained process numbers", message.contains("process numbers"), equalTo(true)); + } + + } + + // test to make sure expressions are not allowed to be used as update scripts + public void testInvalidUpdateScript() throws Exception { + try { + createIndex("test_index"); + ensureGreen("test_index"); + indexRandom(true, client().prepareIndex("test_index", "doc", "1").setSource("text_field", "text")); + UpdateRequestBuilder urb = client().prepareUpdate().setIndex("test_index"); + urb.setType("doc"); + urb.setId("1"); + urb.setScript(new Script("0", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null)); + urb.get(); + fail("Expression scripts should not be allowed to run as update scripts."); + } catch (Exception e) { + String message = e.getMessage(); + assertThat(message + " should have contained failed to execute", message.contains("failed to execute"), equalTo(true)); + message = e.getCause().getMessage(); + assertThat(message + " should have contained not supported", message.contains("not supported"), equalTo(true)); + } + } + + // test to make sure expressions are not allowed to be used as mapping scripts + public void testInvalidMappingScript() throws Exception{ + try { + createIndex("test_index"); + ensureGreen("test_index"); + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + builder.startObject("transform"); + builder.field("script", "1.0"); + builder.field("lang", ExpressionScriptEngineService.NAME); + builder.endObject(); + builder.startObject("properties"); + builder.startObject("double_field"); + builder.field("type", "double"); + builder.endObject(); + builder.endObject(); + builder.endObject(); + client().admin().indices().preparePutMapping("test_index").setType("trans_test").setSource(builder).get(); + client().prepareIndex("test_index", "trans_test", "1").setSource("double_field", 0.0).get(); + fail("Expression scripts should not be allowed to run as mapping scripts."); + } catch (Exception e) { + String message = ExceptionsHelper.detailedMessage(e); + assertThat(message + " should have contained failed to parse", message.contains("failed to parse"), equalTo(true)); + assertThat(message + " should have contained not supported", message.contains("not supported"), equalTo(true)); + } + } + + // test to make sure expressions are allowed to be used for reduce in pipeline aggregations + public void testPipelineAggregationScript() throws Exception { + createIndex("agg_index"); + ensureGreen("agg_index"); + indexRandom(true, + client().prepareIndex("agg_index", "doc", "1").setSource("one", 1.0, "two", 2.0, "three", 3.0, "four", 4.0), + client().prepareIndex("agg_index", "doc", "2").setSource("one", 2.0, "two", 2.0, "three", 3.0, "four", 4.0), + client().prepareIndex("agg_index", "doc", "3").setSource("one", 3.0, "two", 2.0, "three", 3.0, "four", 4.0), + client().prepareIndex("agg_index", "doc", "4").setSource("one", 4.0, "two", 2.0, "three", 3.0, "four", 4.0), + client().prepareIndex("agg_index", "doc", "5").setSource("one", 5.0, "two", 2.0, "three", 3.0, "four", 4.0)); + SearchResponse response = client() + .prepareSearch("agg_index") + .addAggregation( + histogram("histogram") + .field("one") + .interval(2) + .subAggregation(sum("twoSum").field("two")) + .subAggregation(sum("threeSum").field("three")) + .subAggregation(sum("fourSum").field("four")) + .subAggregation( + seriesArithmetic("totalSum").setBucketsPaths("twoSum", "threeSum", "fourSum").script( + new Script("_value0 + _value1 + _value2", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null)))).execute().actionGet(); + + InternalHistogram histogram = response.getAggregations().get("histogram"); + assertThat(histogram, notNullValue()); + assertThat(histogram.getName(), equalTo("histogram")); + List buckets = histogram.getBuckets(); + + for (int bucketCount = 0; bucketCount < buckets.size(); ++bucketCount) { + Histogram.Bucket bucket = buckets.get(bucketCount); + if (bucket.getDocCount() == 1) { + SimpleValue seriesArithmetic = bucket.getAggregations().get("totalSum"); + assertThat(seriesArithmetic, notNullValue()); + double seriesArithmeticValue = seriesArithmetic.value(); + assertEquals(9.0, seriesArithmeticValue, 0.001); + } else if (bucket.getDocCount() == 2) { + SimpleValue seriesArithmetic = bucket.getAggregations().get("totalSum"); + assertThat(seriesArithmetic, notNullValue()); + double seriesArithmeticValue = seriesArithmetic.value(); + assertEquals(18.0, seriesArithmeticValue, 0.001); + } else { + fail("Incorrect number of documents in a bucket in the histogram."); + } + } } }