diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index d16383cb87a..7c5803e19c3 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -1084,13 +1084,6 @@ <suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]MoreExpressionTests.java" checks="LineLength" /> <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovyPlugin.java" checks="LineLength" /> <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovyScriptEngineService.java" checks="LineLength" /> - <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]IPv4RangeTests.java" checks="LineLength" /> - <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]IndexLookupTests.java" checks="LineLength" /> - <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]RandomScoreFunctionTests.java" checks="LineLength" /> - <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]ScriptedMetricTests.java" checks="LineLength" /> - <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]SearchFieldsTests.java" checks="LineLength" /> - <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]SimpleSortTests.java" checks="LineLength" /> - <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]package-info.java" checks="LineLength" /> <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovyScriptTests.java" checks="LineLength" /> <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovySecurityTests.java" checks="LineLength" /> <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]MultiPercolateRequest.java" checks="LineLength" /> diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java index a1aff430f26..e4b80b5dc80 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java @@ -34,7 +34,6 @@ import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.cache.Cache; @@ -101,7 +100,6 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust private final ScriptModes scriptModes; private final ScriptContextRegistry scriptContextRegistry; - private final ParseFieldMatcher parseFieldMatcher; private final ScriptMetrics scriptMetrics = new ScriptMetrics(); private ClusterState clusterState; @@ -113,7 +111,6 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust Objects.requireNonNull(scriptEngineRegistry); Objects.requireNonNull(scriptContextRegistry); Objects.requireNonNull(scriptSettings); - this.parseFieldMatcher = new ParseFieldMatcher(settings); if (Strings.hasLength(settings.get(DISABLE_DYNAMIC_SCRIPTING_SETTING))) { throw new IllegalArgumentException(DISABLE_DYNAMIC_SCRIPTING_SETTING + " is not a supported setting, replace with fine-grained script settings. \n" + "Dynamic scripts can be enabled for all languages and all operations by replacing `script.disable_dynamic: false` with `script.inline: true` and `script.stored: true` in elasticsearch.yml"); diff --git a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java index 41b19a3e572..e315f8d816c 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java @@ -19,11 +19,9 @@ package org.elasticsearch.script; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.script.ScriptService; import java.util.ArrayList; import java.util.Collections; @@ -31,7 +29,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; -import java.util.stream.Collectors; public class ScriptSettings { diff --git a/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java index fdf7049aced..920d2537935 100644 --- a/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java @@ -34,7 +34,7 @@ import static org.hamcrest.Matchers.instanceOf; public class ScriptQueryBuilderTests extends AbstractQueryTestCase<ScriptQueryBuilder> { @Override protected ScriptQueryBuilder doCreateTestQueryBuilder() { - String script = "5"; + String script = "1"; Map<String, Object> params = Collections.emptyMap(); return new ScriptQueryBuilder(new Script(script, ScriptType.INLINE, MockScriptEngine.NAME, params)); } diff --git a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java index 6950a98631f..bd841a05ca1 100644 --- a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java @@ -155,7 +155,7 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi functionBuilder = fieldValueFactorFunctionBuilder; break; case 2: - String script = "5"; + String script = "1"; Map<String, Object> params = Collections.emptyMap(); functionBuilder = new ScriptScoreFunctionBuilder( new Script(script, ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, params)); diff --git a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java index f932317085b..954de105a4f 100644 --- a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java +++ b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java @@ -37,14 +37,16 @@ public class FileScriptTests extends ESTestCase { Path scriptsDir = homeDir.resolve("config").resolve("scripts"); Files.createDirectories(scriptsDir); Path mockscript = scriptsDir.resolve("script1.mockscript"); - Files.write(mockscript, "1".getBytes("UTF-8")); + String scriptSource = "1"; + Files.write(mockscript, scriptSource.getBytes("UTF-8")); settings = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), homeDir) // no file watching, so we don't need a ResourceWatcherService .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false) .put(settings) .build(); - ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singleton(new MockScriptEngine())); + MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap(scriptSource, script -> "1")); + ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singleton(scriptEngine)); ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); return new ScriptService(settings, new Environment(settings), null, scriptEngineRegistry, scriptContextRegistry, scriptSettings); diff --git a/core/src/test/java/org/elasticsearch/script/IndexLookupIT.java b/core/src/test/java/org/elasticsearch/script/IndexLookupIT.java new file mode 100644 index 00000000000..7e57d41acea --- /dev/null +++ b/core/src/test/java/org/elasticsearch/script/IndexLookupIT.java @@ -0,0 +1,1025 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.lookup.IndexField; +import org.elasticsearch.search.lookup.IndexFieldTerm; +import org.elasticsearch.search.lookup.IndexLookup; +import org.elasticsearch.search.lookup.LeafIndexLookup; +import org.elasticsearch.search.lookup.TermPosition; +import org.elasticsearch.test.ESIntegTestCase; +import org.hamcrest.Matchers; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.function.Function; + +import static java.util.Collections.emptyList; +import static org.elasticsearch.script.ScriptService.ScriptType; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + +public class IndexLookupIT extends ESIntegTestCase { + + private static final String INCLUDE_ALL = "_FREQUENCIES|_OFFSETS|_PAYLOADS|_POSITIONS|_CACHE"; + private static final int ALL_FLAGS = IndexLookup.FLAG_FREQUENCIES + | IndexLookup.FLAG_OFFSETS + | IndexLookup.FLAG_PAYLOADS + | IndexLookup.FLAG_POSITIONS + | IndexLookup.FLAG_CACHE; + + private static final String INCLUDE_ALL_BUT_CACHE = "_FREQUENCIES|_OFFSETS|_PAYLOADS|_POSITIONS"; + private static final int ALL_FLAGS_WITHOUT_CACHE = IndexLookup.FLAG_FREQUENCIES + | IndexLookup.FLAG_OFFSETS + | IndexLookup.FLAG_PAYLOADS + | IndexLookup.FLAG_POSITIONS; + + private HashMap<String, List<Object>> expectedEndOffsetsArray; + private HashMap<String, List<Object>> expectedPayloadsArray; + private HashMap<String, List<Object>> expectedPositionsArray; + private HashMap<String, List<Object>> expectedStartOffsetsArray; + + @Override + protected Collection<Class<? extends Plugin>> nodePlugins() { + return Collections.singleton(CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends MockScriptPlugin { + + @Override + @SuppressWarnings("unchecked") + protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() { + Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>(); + + scripts.put("term = _index['int_payload_field']['c']; term.tf()", vars -> tf(vars, "int_payload_field", "c")); + scripts.put("term = _index['int_payload_field']['b']; term.tf()", vars -> tf(vars, "int_payload_field", "b")); + + scripts.put("Sum the payloads of [float_payload_field][b]", vars -> payloadSum(vars, "float_payload_field", "b")); + scripts.put("Sum the payloads of [int_payload_field][b]", vars -> payloadSum(vars, "int_payload_field", "b")); + + scripts.put("createPositionsArrayScriptIterateTwice[b," + INCLUDE_ALL + ",position]", + vars -> createPositionsArrayScriptIterateTwice(vars, "b", ALL_FLAGS, p -> p.position)); + scripts.put("createPositionsArrayScriptIterateTwice[b," + INCLUDE_ALL + ",startOffset]", + vars -> createPositionsArrayScriptIterateTwice(vars, "b", ALL_FLAGS, p -> p.startOffset)); + scripts.put("createPositionsArrayScriptIterateTwice[b," + INCLUDE_ALL + ",endOffset]", + vars -> createPositionsArrayScriptIterateTwice(vars, "b", ALL_FLAGS, p -> p.endOffset)); + scripts.put("createPositionsArrayScriptIterateTwice[b," + INCLUDE_ALL + ",payloadAsInt(-1)]", + vars -> createPositionsArrayScriptIterateTwice(vars, "b", ALL_FLAGS, p -> p.payloadAsInt(-1))); + + scripts.put("createPositionsArrayScriptIterateTwice[b,_FREQUENCIES|_OFFSETS|_PAYLOADS|_POSITIONS,position]", + vars -> createPositionsArrayScriptIterateTwice(vars, "b", ALL_FLAGS_WITHOUT_CACHE, p -> p.position)); + scripts.put("createPositionsArrayScriptIterateTwice[b,_FREQUENCIES|_OFFSETS|_PAYLOADS|_POSITIONS,startOffset]", + vars -> createPositionsArrayScriptIterateTwice(vars, "b", ALL_FLAGS_WITHOUT_CACHE, p -> p.startOffset)); + scripts.put("createPositionsArrayScriptIterateTwice[b,_FREQUENCIES|_OFFSETS|_PAYLOADS|_POSITIONS,endOffset]", + vars -> createPositionsArrayScriptIterateTwice(vars, "b", ALL_FLAGS_WITHOUT_CACHE, p -> p.endOffset)); + scripts.put("createPositionsArrayScriptIterateTwice[b,_FREQUENCIES|_OFFSETS|_PAYLOADS|_POSITIONS,payloadAsInt(-1)]", + vars -> createPositionsArrayScriptIterateTwice(vars, "b", ALL_FLAGS_WITHOUT_CACHE, p -> p.payloadAsInt(-1))); + + scripts.put("createPositionsArrayScriptGetInfoObjectTwice[b,_FREQUENCIES|_OFFSETS|_PAYLOADS|_POSITIONS,position]", + vars -> createPositionsArrayScriptGetInfoObjectTwice(vars, "b", ALL_FLAGS_WITHOUT_CACHE, p -> p.position)); + scripts.put("createPositionsArrayScriptGetInfoObjectTwice[b,_FREQUENCIES|_OFFSETS|_PAYLOADS|_POSITIONS,startOffset]", + vars -> createPositionsArrayScriptGetInfoObjectTwice(vars, "b", ALL_FLAGS_WITHOUT_CACHE, p -> p.startOffset)); + scripts.put("createPositionsArrayScriptGetInfoObjectTwice[b,_FREQUENCIES|_OFFSETS|_PAYLOADS|_POSITIONS,endOffset]", + vars -> createPositionsArrayScriptGetInfoObjectTwice(vars, "b", ALL_FLAGS_WITHOUT_CACHE, p -> p.endOffset)); + scripts.put("createPositionsArrayScriptGetInfoObjectTwice[b,_FREQUENCIES|_OFFSETS|_PAYLOADS|_POSITIONS,payloadAsInt(-1)]", + vars -> createPositionsArrayScriptGetInfoObjectTwice(vars, "b", ALL_FLAGS_WITHOUT_CACHE, p -> p.payloadAsInt(-1))); + + scripts.put("createPositionsArrayScript[int_payload_field,b,_POSITIONS,position]", + vars -> createPositionsArrayScript(vars, "int_payload_field", "b", IndexLookup.FLAG_POSITIONS, p -> p.position)); + + scripts.put("createPositionsArrayScript[int_payload_field,b,_OFFSETS,position]", + vars -> createPositionsArrayScript(vars, "int_payload_field", "b", IndexLookup.FLAG_OFFSETS, p -> p.position)); + scripts.put("createPositionsArrayScript[int_payload_field,b,_OFFSETS,startOffset]", + vars -> createPositionsArrayScript(vars, "int_payload_field", "b", IndexLookup.FLAG_OFFSETS, p -> p.startOffset)); + scripts.put("createPositionsArrayScript[int_payload_field,b,_OFFSETS,endOffset]", + vars -> createPositionsArrayScript(vars, "int_payload_field", "b", IndexLookup.FLAG_OFFSETS, p -> p.endOffset)); + scripts.put("createPositionsArrayScript[int_payload_field,b,_OFFSETS,payloadAsInt(-1)]", + vars -> createPositionsArrayScript(vars, "int_payload_field", "b", IndexLookup.FLAG_OFFSETS, p -> p.payloadAsInt(-1))); + + scripts.put("createPositionsArrayScript[int_payload_field,b,_PAYLOADS,position]", + vars -> createPositionsArrayScript(vars, "int_payload_field", "b", IndexLookup.FLAG_PAYLOADS, p -> p.position)); + scripts.put("createPositionsArrayScript[int_payload_field,b,_PAYLOADS,startOffset]", + vars -> createPositionsArrayScript(vars, "int_payload_field", "b", IndexLookup.FLAG_PAYLOADS, p -> p.startOffset)); + scripts.put("createPositionsArrayScript[int_payload_field,b,_PAYLOADS,endOffset]", + vars -> createPositionsArrayScript(vars, "int_payload_field", "b", IndexLookup.FLAG_PAYLOADS, p -> p.endOffset)); + scripts.put("createPositionsArrayScript[int_payload_field,b,_PAYLOADS,payloadAsInt(-1)]", + vars -> createPositionsArrayScript(vars, "int_payload_field", "b", IndexLookup.FLAG_PAYLOADS, p -> p.payloadAsInt(-1))); + + int posoffpay = IndexLookup.FLAG_POSITIONS|IndexLookup.FLAG_OFFSETS|IndexLookup.FLAG_PAYLOADS; + scripts.put("createPositionsArrayScript[int_payload_field,b,_POSITIONS|_OFFSETS|_PAYLOADS,position]", + vars -> createPositionsArrayScript(vars, "int_payload_field", "b", posoffpay, p -> p.position)); + scripts.put("createPositionsArrayScript[int_payload_field,b,_POSITIONS|_OFFSETS|_PAYLOADS,startOffset]", + vars -> createPositionsArrayScript(vars, "int_payload_field", "b", posoffpay, p -> p.startOffset)); + scripts.put("createPositionsArrayScript[int_payload_field,b,_POSITIONS|_OFFSETS|_PAYLOADS,endOffset]", + vars -> createPositionsArrayScript(vars, "int_payload_field", "b", posoffpay, p -> p.endOffset)); + scripts.put("createPositionsArrayScript[int_payload_field,b,_POSITIONS|_OFFSETS|_PAYLOADS,payloadAsInt(-1)]", + vars -> createPositionsArrayScript(vars, "int_payload_field", "b", posoffpay, p -> p.payloadAsInt(-1))); + + scripts.put("createPositionsArrayScript[int_payload_field,b,_FREQUENCIES|_OFFSETS|_PAYLOADS|_POSITIONS,position]", + vars -> createPositionsArrayScript(vars, "int_payload_field", "b", ALL_FLAGS_WITHOUT_CACHE, p -> p.position)); + scripts.put("createPositionsArrayScript[int_payload_field,b,_FREQUENCIES|_OFFSETS|_PAYLOADS|_POSITIONS,startOffset]", + vars -> createPositionsArrayScript(vars, "int_payload_field", "b", ALL_FLAGS_WITHOUT_CACHE, p -> p.startOffset)); + scripts.put("createPositionsArrayScript[int_payload_field,b,_FREQUENCIES|_OFFSETS|_PAYLOADS|_POSITIONS,endOffset]", + vars -> createPositionsArrayScript(vars, "int_payload_field", "b", ALL_FLAGS_WITHOUT_CACHE, p -> p.endOffset)); + scripts.put("createPositionsArrayScript[int_payload_field,b,_FREQUENCIES|_OFFSETS|_PAYLOADS|_POSITIONS,payloadAsInt(-1)]", + vars -> createPositionsArrayScript(vars, "int_payload_field", "b", ALL_FLAGS_WITHOUT_CACHE, p -> p.payloadAsInt(-1))); + + scripts.put("createPositionsArrayScript" + + "[float_payload_field,b," + INCLUDE_ALL + ",payloadAsFloat(-1)]", + vars -> createPositionsArrayScript(vars,"float_payload_field", "b", ALL_FLAGS, p -> p.payloadAsFloat(-1))); + scripts.put("createPositionsArrayScript" + + "[string_payload_field,b," + INCLUDE_ALL + ",payloadAsString()]", + vars -> createPositionsArrayScript(vars,"string_payload_field", "b", ALL_FLAGS, TermPosition::payloadAsString)); + scripts.put("createPositionsArrayScript" + + "[int_payload_field,c," + INCLUDE_ALL + ",payloadAsInt(-1)]", + vars -> createPositionsArrayScript(vars,"int_payload_field", "c", ALL_FLAGS, p -> p.payloadAsInt(-1))); + + // Call with different flags twice, equivalent to: + // term = _index['int_payload_field']['b']; return _index['int_payload_field'].get('b', _POSITIONS).tf(); + scripts.put("Call with different flags twice", vars -> { + LeafIndexLookup leafIndexLookup = (LeafIndexLookup) vars.get("_index"); + IndexField indexField = leafIndexLookup.get("int_payload_field"); + + // 1st call + indexField.get("b"); + try { + // 2nd call, must throws an exception + return indexField.get("b", IndexLookup.FLAG_POSITIONS).tf(); + } catch (IOException e) { + throw new ScriptException(e.getMessage(), e, emptyList(), "Call with different flags twice", CustomScriptPlugin.NAME); + } + }); + + // Call with same flags twice: equivalent to: + // term = _index['int_payload_field'].get('b', _POSITIONS | _FREQUENCIES);return _index['int_payload_field']['b'].tf(); + scripts.put("Call with same flags twice", vars -> { + LeafIndexLookup leafIndexLookup = (LeafIndexLookup) vars.get("_index"); + IndexField indexField = leafIndexLookup.get("int_payload_field"); + + // 1st call + indexField.get("b", IndexLookup.FLAG_POSITIONS | IndexLookup.FLAG_FREQUENCIES); + try { + // 2nd call, must throws an exception + return indexField.get("b").tf(); + } catch (IOException e) { + throw new ScriptException(e.getMessage(), e, emptyList(), "Call with same flags twice", CustomScriptPlugin.NAME); + } + }); + + // get the number of all docs + scripts.put("_index.numDocs()", + vars -> ((LeafIndexLookup) vars.get("_index")).numDocs()); + + // get the number of docs with field float_payload_field + scripts.put("_index['float_payload_field'].docCount()", + vars -> indexFieldScript(vars, "float_payload_field", indexField -> { + try { + return indexField.docCount(); + } catch (IOException e) { + throw new ScriptException(e.getMessage(), e, emptyList(), "docCount()", CustomScriptPlugin.NAME); + } + })); + + // corner case: what if the field does not exist? + scripts.put("_index['non_existent_field'].docCount()", + vars -> indexFieldScript(vars, "non_existent_field", indexField -> { + try { + return indexField.docCount(); + } catch (IOException e) { + throw new ScriptException(e.getMessage(), e, emptyList(), "docCount()", CustomScriptPlugin.NAME); + } + })); + + // get the number of all tokens in all docs + scripts.put("_index['float_payload_field'].sumttf()", + vars -> indexFieldScript(vars, "float_payload_field", indexField -> { + try { + return indexField.sumttf(); + } catch (IOException e) { + throw new ScriptException(e.getMessage(), e, emptyList(), "sumttf()", CustomScriptPlugin.NAME); + } + })); + + // corner case get the number of all tokens in all docs for non existent + // field + scripts.put("_index['non_existent_field'].sumttf()", + vars -> indexFieldScript(vars, "non_existent_field", indexField -> { + try { + return indexField.sumttf(); + } catch (IOException e) { + throw new ScriptException(e.getMessage(), e, emptyList(), "sumttf()", CustomScriptPlugin.NAME); + } + })); + + // get the sum of doc freqs in all docs + scripts.put("_index['float_payload_field'].sumdf()", + vars -> indexFieldScript(vars, "float_payload_field", indexField -> { + try { + return indexField.sumdf(); + } catch (IOException e) { + throw new ScriptException(e.getMessage(), e, emptyList(), "sumdf()", CustomScriptPlugin.NAME); + } + })); + + // get the sum of doc freqs in all docs for non existent field + scripts.put("_index['non_existent_field'].sumdf()", + vars -> indexFieldScript(vars, "non_existent_field", indexField -> { + try { + return indexField.sumdf(); + } catch (IOException e) { + throw new ScriptException(e.getMessage(), e, emptyList(), "sumdf()", CustomScriptPlugin.NAME); + } + })); + + // check term frequencies for 'a' + scripts.put("term = _index['float_payload_field']['a']; if (term != null) {term.tf()}", + vars -> indexFieldTermScript(vars, "float_payload_field", "a", indexFieldTerm -> { + try { + if (indexFieldTerm != null) { + return indexFieldTerm.tf(); + } + } catch (IOException e) { + throw new ScriptException(e.getMessage(), e, emptyList(), "term.tf()", CustomScriptPlugin.NAME); + } + return null; + })); + + // check doc frequencies for 'c' + scripts.put("term = _index['float_payload_field']['c']; if (term != null) {term.df()}", + vars -> indexFieldTermScript(vars, "float_payload_field", "c", indexFieldTerm -> { + try { + if (indexFieldTerm != null) { + return indexFieldTerm.df(); + } + } catch (IOException e) { + throw new ScriptException(e.getMessage(), e, emptyList(), "term.df()", CustomScriptPlugin.NAME); + } + return null; + })); + + // check doc frequencies for term that does not exist + scripts.put("term = _index['float_payload_field']['non_existent_term']; if (term != null) {term.df()}", + vars -> indexFieldTermScript(vars, "float_payload_field", "non_existent_term", indexFieldTerm -> { + try { + if (indexFieldTerm != null) { + return indexFieldTerm.df(); + } + } catch (IOException e) { + throw new ScriptException(e.getMessage(), e, emptyList(), "term.df()", CustomScriptPlugin.NAME); + } + return null; + })); + + // check doc frequencies for term that does not exist + scripts.put("term = _index['non_existent_field']['non_existent_term']; if (term != null) {term.tf()}", + vars -> indexFieldTermScript(vars, "non_existent_field", "non_existent_term", indexFieldTerm -> { + try { + if (indexFieldTerm != null) { + return indexFieldTerm.tf(); + } + } catch (IOException e) { + throw new ScriptException(e.getMessage(), e, emptyList(), "term.tf()", CustomScriptPlugin.NAME); + } + return null; + })); + + // check total term frequencies for 'a' + scripts.put("term = _index['float_payload_field']['a']; if (term != null) {term.ttf()}", + vars -> indexFieldTermScript(vars, "float_payload_field", "a", indexFieldTerm -> { + try { + if (indexFieldTerm != null) { + return indexFieldTerm.ttf(); + } + } catch (IOException e) { + throw new ScriptException(e.getMessage(), e, emptyList(), "term.ttf()", CustomScriptPlugin.NAME); + } + return null; + })); + + return scripts; + } + + @SuppressWarnings("unchecked") + static Object indexFieldScript(Map<String, Object> vars, String fieldName, Function<IndexField, Object> fn) { + LeafIndexLookup leafIndexLookup = (LeafIndexLookup) vars.get("_index"); + return fn.apply(leafIndexLookup.get(fieldName)); + } + + @SuppressWarnings("unchecked") + static Object indexFieldTermScript(Map<String, Object> vars, String fieldName, String term, Function<IndexFieldTerm, Object> fn) { + return indexFieldScript(vars, fieldName, indexField -> fn.apply(indexField.get(term))); + } + + @SuppressWarnings("unchecked") + static Object tf(Map<String, Object> vars, String fieldName, String term) { + return indexFieldTermScript(vars, fieldName, term, indexFieldTerm -> { + try { + return indexFieldTerm.tf(); + } catch (IOException e) { + throw new RuntimeException("Mocked script error when retrieving TF for [" + fieldName + "][" + term + "]"); + } + }); + } + + // Sum the payloads for a given field term, equivalent to: + // term = _index['float_payload_field'].get('b', _FREQUENCIES|_OFFSETS|_PAYLOADS|_POSITIONS|_CACHE); + // payloadSum=0; + // for (pos in term) { + // payloadSum += pos.payloadAsInt(0) + // }; + // return payloadSum; + @SuppressWarnings("unchecked") + static Object payloadSum(Map<String, Object> vars, String fieldName, String term) { + return indexFieldScript(vars, fieldName, indexField -> { + IndexFieldTerm indexFieldTerm = indexField.get(term, IndexLookup.FLAG_FREQUENCIES + | IndexLookup.FLAG_OFFSETS + | IndexLookup.FLAG_PAYLOADS + | IndexLookup.FLAG_POSITIONS + | IndexLookup.FLAG_CACHE); + int payloadSum = 0; + for (TermPosition position : indexFieldTerm) { + payloadSum += position.payloadAsInt(0); + } + return payloadSum; + }); + } + + @SuppressWarnings("unchecked") + static List<Object> createPositionsArrayScriptGetInfoObjectTwice(Map<String, Object> vars, String term, int flags, + Function<TermPosition, Object> fn) { + LeafIndexLookup leafIndexLookup = (LeafIndexLookup) vars.get("_index"); + IndexField indexField = leafIndexLookup.get("int_payload_field"); + + // 1st call + IndexFieldTerm indexFieldTerm = indexField.get(term, flags); + + List<Object> array = new ArrayList<>(); + for (TermPosition position : indexFieldTerm) { + array.add(fn.apply(position)); + } + + // 2nd call + indexField.get(term, flags); + + array = new ArrayList<>(); + for (TermPosition position : indexFieldTerm) { + array.add(fn.apply(position)); + } + + return array; + } + + @SuppressWarnings("unchecked") + static List<Object> createPositionsArrayScriptIterateTwice(Map<String, Object> vars, String term, int flags, + Function<TermPosition, Object> fn) { + LeafIndexLookup leafIndexLookup = (LeafIndexLookup) vars.get("_index"); + IndexField indexField = leafIndexLookup.get("int_payload_field"); + + IndexFieldTerm indexFieldTerm = indexField.get(term, flags); + + // 1st iteration + List<Object> array = new ArrayList<>(); + for (TermPosition position : indexFieldTerm) { + array.add(fn.apply(position)); + } + + // 2nd iteration + array = new ArrayList<>(); + for (TermPosition position : indexFieldTerm) { + array.add(fn.apply(position)); + } + + return array; + } + + @SuppressWarnings("unchecked") + static List<Object> createPositionsArrayScript(Map<String, Object> vars, String field, String term, int flags, + Function<TermPosition, Object> fn) { + + LeafIndexLookup leafIndexLookup = (LeafIndexLookup) vars.get("_index"); + IndexField indexField = leafIndexLookup.get(field); + + IndexFieldTerm indexFieldTerm = indexField.get(term, flags); + List<Object> array = new ArrayList<>(); + for (TermPosition position : indexFieldTerm) { + array.add(fn.apply(position)); + } + return array; + } + } + + void initTestData() throws InterruptedException, ExecutionException, IOException { + HashMap<String, List<Object>> emptyArray = new HashMap<>(); + List<Object> empty1 = new ArrayList<>(); + empty1.add(-1); + empty1.add(-1); + emptyArray.put("1", empty1); + List<Object> empty2 = new ArrayList<>(); + empty2.add(-1); + empty2.add(-1); + emptyArray.put("2", empty2); + List<Object> empty3 = new ArrayList<>(); + empty3.add(-1); + empty3.add(-1); + emptyArray.put("3", empty3); + + expectedPositionsArray = new HashMap<>(); + + List<Object> pos1 = new ArrayList<>(); + pos1.add(1); + pos1.add(2); + expectedPositionsArray.put("1", pos1); + List<Object> pos2 = new ArrayList<>(); + pos2.add(0); + pos2.add(1); + expectedPositionsArray.put("2", pos2); + List<Object> pos3 = new ArrayList<>(); + pos3.add(0); + pos3.add(4); + expectedPositionsArray.put("3", pos3); + + expectedPayloadsArray = new HashMap<>(); + List<Object> pay1 = new ArrayList<>(); + pay1.add(2); + pay1.add(3); + expectedPayloadsArray.put("1", pay1); + List<Object> pay2 = new ArrayList<>(); + pay2.add(1); + pay2.add(2); + expectedPayloadsArray.put("2", pay2); + List<Object> pay3 = new ArrayList<>(); + pay3.add(1); + pay3.add(-1); + expectedPayloadsArray.put("3", pay3); + /* + * "a|1 b|2 b|3 c|4 d " "b|1 b|2 c|3 d|4 a " "b|1 c|2 d|3 a|4 b " + */ + expectedStartOffsetsArray = new HashMap<>(); + List<Object> starts1 = new ArrayList<>(); + starts1.add(4); + starts1.add(8); + expectedStartOffsetsArray.put("1", starts1); + List<Object> starts2 = new ArrayList<>(); + starts2.add(0); + starts2.add(4); + expectedStartOffsetsArray.put("2", starts2); + List<Object> starts3 = new ArrayList<>(); + starts3.add(0); + starts3.add(16); + expectedStartOffsetsArray.put("3", starts3); + + expectedEndOffsetsArray = new HashMap<>(); + List<Object> ends1 = new ArrayList<>(); + ends1.add(7); + ends1.add(11); + expectedEndOffsetsArray.put("1", ends1); + List<Object> ends2 = new ArrayList<>(); + ends2.add(3); + ends2.add(7); + expectedEndOffsetsArray.put("2", ends2); + List<Object> ends3 = new ArrayList<>(); + ends3.add(3); + ends3.add(17); + expectedEndOffsetsArray.put("3", ends3); + + XContentBuilder mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("type1") + .startObject("properties") + .startObject("int_payload_field") + .field("type", "text") + .field("index_options", "offsets") + .field("analyzer", "payload_int") + .endObject() + .endObject() + .endObject() + .endObject(); + + assertAcked(prepareCreate("test").addMapping("type1", mapping).setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.analysis.analyzer.payload_int.tokenizer", "whitespace") + .putArray("index.analysis.analyzer.payload_int.filter", "delimited_int") + .put("index.analysis.filter.delimited_int.delimiter", "|") + .put("index.analysis.filter.delimited_int.encoding", "int") + .put("index.analysis.filter.delimited_int.type", "delimited_payload_filter"))); + indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("int_payload_field", "a|1 b|2 b|3 c|4 d "), client() + .prepareIndex("test", "type1", "2").setSource("int_payload_field", "b|1 b|2 c|3 d|4 a "), + client().prepareIndex("test", "type1", "3").setSource("int_payload_field", "b|1 c|2 d|3 a|4 b ")); + ensureGreen(); + } + + public void testTwoScripts() throws Exception { + initTestData(); + + Script scriptFieldScript = createScript("term = _index['int_payload_field']['c']; term.tf()"); + Script scoreScript = createScript("term = _index['int_payload_field']['b']; term.tf()"); + Map<String, Object> expectedResultsField = new HashMap<>(); + expectedResultsField.put("1", 1); + expectedResultsField.put("2", 1); + expectedResultsField.put("3", 1); + Map<String, Object> expectedResultsScore = new HashMap<>(); + expectedResultsScore.put("1", 2f); + expectedResultsScore.put("2", 2f); + expectedResultsScore.put("3", 2f); + checkOnlyFunctionScore(scoreScript, expectedResultsScore, 3); + checkValueInEachDocWithFunctionScore(scriptFieldScript, expectedResultsField, scoreScript, expectedResultsScore, 3); + + } + + public void testCallWithDifferentFlagsFails() throws Exception { + initTestData(); + final int numPrimaries = getNumShards("test").numPrimaries; + final String expectedError = "You must call get with all required flags! " + + "Instead of _index['int_payload_field'].get('b', _FREQUENCIES) and _index['int_payload_field'].get('b', _POSITIONS)" + + " call _index['int_payload_field'].get('b', _FREQUENCIES | _POSITIONS) once]"; + + // should throw an exception, we cannot call with different flags twice + // if the flags of the second call were not included in the first call. + Script script = createScript("Call with different flags twice"); + try { + SearchResponse response = client().prepareSearch("test") + .setQuery(QueryBuilders.matchAllQuery()) + .addScriptField("tvtest", script) + .get(); + + assertThat(numPrimaries, greaterThan(1)); + assertThat(response.getFailedShards(), greaterThanOrEqualTo(1)); + + for (ShardSearchFailure failure : response.getShardFailures()) { + assertThat(failure.reason(), containsString(expectedError)); + } + } catch (SearchPhaseExecutionException e) { + assertThat(numPrimaries, equalTo(1)); + assertThat(e.toString(), containsString(expectedError)); + } + + // Should not throw an exception this way round + script = createScript("Call with same flags twice"); + assertThat(client().prepareSearch("test") + .setQuery(QueryBuilders.matchAllQuery()) + .addScriptField("tvtest", script) + .get().getHits().getTotalHits(), greaterThan(0L)); + } + + private void checkOnlyFunctionScore(Script scoreScript, Map<String, Object> expectedScore, int numExpectedDocs) { + SearchResponse sr = client().prepareSearch("test") + .setQuery(QueryBuilders.functionScoreQuery(ScoreFunctionBuilders.scriptFunction(scoreScript))).execute() + .actionGet(); + assertHitCount(sr, numExpectedDocs); + for (SearchHit hit : sr.getHits().getHits()) { + assertThat("for doc " + hit.getId(), ((Float) expectedScore.get(hit.getId())).doubleValue(), + Matchers.closeTo(hit.score(), 1.e-4)); + } + } + + public void testDocumentationExample() throws Exception { + initTestData(); + + Script script = createScript("Sum the payloads of [float_payload_field][b]"); + + // non existing field: sum should be 0 + HashMap<String, Object> zeroArray = new HashMap<>(); + zeroArray.put("1", 0); + zeroArray.put("2", 0); + zeroArray.put("3", 0); + checkValueInEachDoc(script, zeroArray, 3); + + script = createScript("Sum the payloads of [int_payload_field][b]"); + + // existing field: sums should be as here: + zeroArray.put("1", 5); + zeroArray.put("2", 3); + zeroArray.put("3", 1); + checkValueInEachDoc(script, zeroArray, 3); + } + + public void testIteratorAndRecording() throws Exception { + initTestData(); + + // call twice with record: should work as expected + Script script = createPositionsArrayScriptIterateTwice("b", INCLUDE_ALL, "position"); + checkArrayValsInEachDoc(script, expectedPositionsArray, 3); + script = createPositionsArrayScriptIterateTwice("b", INCLUDE_ALL, "startOffset"); + checkArrayValsInEachDoc(script, expectedStartOffsetsArray, 3); + script = createPositionsArrayScriptIterateTwice("b", INCLUDE_ALL, "endOffset"); + checkArrayValsInEachDoc(script, expectedEndOffsetsArray, 3); + script = createPositionsArrayScriptIterateTwice("b", INCLUDE_ALL, "payloadAsInt(-1)"); + checkArrayValsInEachDoc(script, expectedPayloadsArray, 3); + + // no record and get iterator twice: should fail + script = createPositionsArrayScriptIterateTwice("b", INCLUDE_ALL_BUT_CACHE, "position"); + checkExceptions(script); + script = createPositionsArrayScriptIterateTwice("b", INCLUDE_ALL_BUT_CACHE, "startOffset"); + checkExceptions(script); + script = createPositionsArrayScriptIterateTwice("b", INCLUDE_ALL_BUT_CACHE, "endOffset"); + checkExceptions(script); + script = createPositionsArrayScriptIterateTwice("b", INCLUDE_ALL_BUT_CACHE, "payloadAsInt(-1)"); + checkExceptions(script); + + // no record and get termObject twice and iterate: should fail + script = createPositionsArrayScriptGetInfoObjectTwice("b", INCLUDE_ALL_BUT_CACHE, "position"); + checkExceptions(script); + script = createPositionsArrayScriptGetInfoObjectTwice("b", INCLUDE_ALL_BUT_CACHE, "startOffset"); + checkExceptions(script); + script = createPositionsArrayScriptGetInfoObjectTwice("b", INCLUDE_ALL_BUT_CACHE, "endOffset"); + checkExceptions(script); + script = createPositionsArrayScriptGetInfoObjectTwice("b", INCLUDE_ALL_BUT_CACHE, "payloadAsInt(-1)"); + checkExceptions(script); + + } + + private Script createPositionsArrayScriptGetInfoObjectTwice(String term, String flags, String what) { + return createScript("createPositionsArrayScriptGetInfoObjectTwice[" + term + "," + flags + "," + what + "]"); + } + + private Script createPositionsArrayScriptIterateTwice(String term, String flags, String what) { + return createScript("createPositionsArrayScriptIterateTwice[" + term + "," + flags + "," + what + "]"); + } + + private Script createPositionsArrayScript(String field, String term, String flags, String what) { + return createScript("createPositionsArrayScript[" + field + "," + term + "," + flags + "," + what + "]"); + } + + private Script createPositionsArrayScriptDefaultGet(String field, String term, String what) { + return createScript("createPositionsArrayScriptDefaultGet[" + field + "," + term + "," + what + "]"); + } + + private Script createScript(String script) { + return new Script(script, ScriptType.INLINE, CustomScriptPlugin.NAME, null); + } + + public void testFlags() throws Exception { + initTestData(); + + // check default flag + Script script = createPositionsArrayScriptDefaultGet("int_payload_field", "b", "position"); + // there should be no positions + /* TODO: the following tests fail with the new postings enum apis because of a bogus assert in BlockDocsEnum + checkArrayValsInEachDoc(script, emptyArray, 3); + script = createPositionsArrayScriptDefaultGet("int_payload_field", "b", "startOffset"); + // there should be no offsets + checkArrayValsInEachDoc(script, emptyArray, 3); + script = createPositionsArrayScriptDefaultGet("int_payload_field", "b", "endOffset"); + // there should be no offsets + checkArrayValsInEachDoc(script, emptyArray, 3); + script = createPositionsArrayScriptDefaultGet("int_payload_field", "b", "payloadAsInt(-1)"); + // there should be no payload + checkArrayValsInEachDoc(script, emptyArray, 3); + + // check FLAG_FREQUENCIES flag + script = createPositionsArrayScript("int_payload_field", "b", "_FREQUENCIES", "position"); + // there should be no positions + checkArrayValsInEachDoc(script, emptyArray, 3); + script = createPositionsArrayScript("int_payload_field", "b", "_FREQUENCIES", "startOffset"); + // there should be no offsets + checkArrayValsInEachDoc(script, emptyArray, 3); + script = createPositionsArrayScript("int_payload_field", "b", "_FREQUENCIES", "endOffset"); + // there should be no offsets + checkArrayValsInEachDoc(script, emptyArray, 3); + script = createPositionsArrayScript("int_payload_field", "b", "_FREQUENCIES", "payloadAsInt(-1)"); + // there should be no payloads + checkArrayValsInEachDoc(script, emptyArray, 3);*/ + + // check FLAG_POSITIONS flag + script = createPositionsArrayScript("int_payload_field", "b", "_POSITIONS", "position"); + // there should be positions + checkArrayValsInEachDoc(script, expectedPositionsArray, 3); + /* TODO: these tests make a bogus assumption that asking for positions will return only positions + script = createPositionsArrayScript("int_payload_field", "b", "_POSITIONS", "startOffset"); + // there should be no offsets + checkArrayValsInEachDoc(script, emptyArray, 3); + script = createPositionsArrayScript("int_payload_field", "b", "_POSITIONS", "endOffset"); + // there should be no offsets + checkArrayValsInEachDoc(script, emptyArray, 3); + script = createPositionsArrayScript("int_payload_field", "b", "_POSITIONS", "payloadAsInt(-1)"); + // there should be no payloads + checkArrayValsInEachDoc(script, emptyArray, 3);*/ + + // check FLAG_OFFSETS flag + script = createPositionsArrayScript("int_payload_field", "b", "_OFFSETS", "position"); + // there should be positions and s forth ... + checkArrayValsInEachDoc(script, expectedPositionsArray, 3); + script = createPositionsArrayScript("int_payload_field", "b", "_OFFSETS", "startOffset"); + checkArrayValsInEachDoc(script, expectedStartOffsetsArray, 3); + script = createPositionsArrayScript("int_payload_field", "b", "_OFFSETS", "endOffset"); + checkArrayValsInEachDoc(script, expectedEndOffsetsArray, 3); + script = createPositionsArrayScript("int_payload_field", "b", "_OFFSETS", "payloadAsInt(-1)"); + checkArrayValsInEachDoc(script, expectedPayloadsArray, 3); + + // check FLAG_PAYLOADS flag + script = createPositionsArrayScript("int_payload_field", "b", "_PAYLOADS", "position"); + checkArrayValsInEachDoc(script, expectedPositionsArray, 3); + script = createPositionsArrayScript("int_payload_field", "b", "_PAYLOADS", "startOffset"); + checkArrayValsInEachDoc(script, expectedStartOffsetsArray, 3); + script = createPositionsArrayScript("int_payload_field", "b", "_PAYLOADS", "endOffset"); + checkArrayValsInEachDoc(script, expectedEndOffsetsArray, 3); + script = createPositionsArrayScript("int_payload_field", "b", "_PAYLOADS", "payloadAsInt(-1)"); + checkArrayValsInEachDoc(script, expectedPayloadsArray, 3); + + // check all flags + String allFlags = "_POSITIONS|_OFFSETS|_PAYLOADS"; + script = createPositionsArrayScript("int_payload_field", "b", allFlags, "position"); + checkArrayValsInEachDoc(script, expectedPositionsArray, 3); + script = createPositionsArrayScript("int_payload_field", "b", allFlags, "startOffset"); + checkArrayValsInEachDoc(script, expectedStartOffsetsArray, 3); + script = createPositionsArrayScript("int_payload_field", "b", allFlags, "endOffset"); + checkArrayValsInEachDoc(script, expectedEndOffsetsArray, 3); + script = createPositionsArrayScript("int_payload_field", "b", allFlags, "payloadAsInt(-1)"); + checkArrayValsInEachDoc(script, expectedPayloadsArray, 3); + + // check all flags without record + script = createPositionsArrayScript("int_payload_field", "b", INCLUDE_ALL_BUT_CACHE, "position"); + checkArrayValsInEachDoc(script, expectedPositionsArray, 3); + script = createPositionsArrayScript("int_payload_field", "b", INCLUDE_ALL_BUT_CACHE, "startOffset"); + checkArrayValsInEachDoc(script, expectedStartOffsetsArray, 3); + script = createPositionsArrayScript("int_payload_field", "b", INCLUDE_ALL_BUT_CACHE, "endOffset"); + checkArrayValsInEachDoc(script, expectedEndOffsetsArray, 3); + script = createPositionsArrayScript("int_payload_field", "b", INCLUDE_ALL_BUT_CACHE, "payloadAsInt(-1)"); + checkArrayValsInEachDoc(script, expectedPayloadsArray, 3); + + } + + private void checkArrayValsInEachDoc(Script script, HashMap<String, List<Object>> expectedArray, int expectedHitSize) { + SearchResponse sr = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()).addScriptField("tvtest", script) + .execute().actionGet(); + assertHitCount(sr, expectedHitSize); + int nullCounter = 0; + for (SearchHit hit : sr.getHits().getHits()) { + Object result = hit.getFields().get("tvtest").getValues(); + Object expectedResult = expectedArray.get(hit.getId()); + assertThat("for doc " + hit.getId(), result, equalTo(expectedResult)); + if (expectedResult != null) { + nullCounter++; + } + } + assertThat(nullCounter, equalTo(expectedArray.size())); + } + + public void testAllExceptPosAndOffset() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") + .startObject("float_payload_field").field("type", "text").field("index_options", "offsets").field("term_vector", "no") + .field("analyzer", "payload_float").endObject().startObject("string_payload_field").field("type", "text") + .field("index_options", "offsets").field("term_vector", "no").field("analyzer", "payload_string").endObject() + .startObject("int_payload_field").field("type", "text").field("index_options", "offsets") + .field("analyzer", "payload_int").endObject().endObject().endObject().endObject(); + assertAcked(prepareCreate("test").addMapping("type1", mapping).setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.analysis.analyzer.payload_float.tokenizer", "whitespace") + .putArray("index.analysis.analyzer.payload_float.filter", "delimited_float") + .put("index.analysis.filter.delimited_float.delimiter", "|") + .put("index.analysis.filter.delimited_float.encoding", "float") + .put("index.analysis.filter.delimited_float.type", "delimited_payload_filter") + .put("index.analysis.analyzer.payload_string.tokenizer", "whitespace") + .putArray("index.analysis.analyzer.payload_string.filter", "delimited_string") + .put("index.analysis.filter.delimited_string.delimiter", "|") + .put("index.analysis.filter.delimited_string.encoding", "identity") + .put("index.analysis.filter.delimited_string.type", "delimited_payload_filter") + .put("index.analysis.analyzer.payload_int.tokenizer", "whitespace") + .putArray("index.analysis.analyzer.payload_int.filter", "delimited_int") + .put("index.analysis.filter.delimited_int.delimiter", "|") + .put("index.analysis.filter.delimited_int.encoding", "int") + .put("index.analysis.filter.delimited_int.type", "delimited_payload_filter") + .put("index.number_of_shards", 1))); + indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("float_payload_field", "a|1 b|2 a|3 b "), client() + .prepareIndex("test", "type1", "2").setSource("string_payload_field", "a|a b|b a|a b "), + client().prepareIndex("test", "type1", "3").setSource("float_payload_field", "a|4 b|5 a|6 b "), + client().prepareIndex("test", "type1", "4").setSource("string_payload_field", "a|b b|a a|b b "), + client().prepareIndex("test", "type1", "5").setSource("float_payload_field", "c "), + client().prepareIndex("test", "type1", "6").setSource("int_payload_field", "c|1")); + + // get the number of all docs + Script script = createScript("_index.numDocs()"); + checkValueInEachDoc(6, script, 6); + + // get the number of docs with field float_payload_field + script = createScript("_index['float_payload_field'].docCount()"); + checkValueInEachDoc(3, script, 6); + + // corner case: what if the field does not exist? + script = createScript("_index['non_existent_field'].docCount()"); + checkValueInEachDoc(0, script, 6); + + // get the number of all tokens in all docs + script = createScript("_index['float_payload_field'].sumttf()"); + checkValueInEachDoc(9, script, 6); + + // corner case get the number of all tokens in all docs for non existent + // field + script = createScript("_index['non_existent_field'].sumttf()"); + checkValueInEachDoc(0, script, 6); + + // get the sum of doc freqs in all docs + script = createScript("_index['float_payload_field'].sumdf()"); + checkValueInEachDoc(5, script, 6); + + // get the sum of doc freqs in all docs for non existent field + script = createScript("_index['non_existent_field'].sumdf()"); + checkValueInEachDoc(0, script, 6); + + // check term frequencies for 'a' + script = createScript("term = _index['float_payload_field']['a']; if (term != null) {term.tf()}"); + Map<String, Object> expectedResults = new HashMap<>(); + expectedResults.put("1", 2); + expectedResults.put("2", 0); + expectedResults.put("3", 2); + expectedResults.put("4", 0); + expectedResults.put("5", 0); + expectedResults.put("6", 0); + checkValueInEachDoc(script, expectedResults, 6); + expectedResults.clear(); + + // check doc frequencies for 'c' + script = createScript("term = _index['float_payload_field']['c']; if (term != null) {term.df()}"); + expectedResults.put("1", 1L); + expectedResults.put("2", 1L); + expectedResults.put("3", 1L); + expectedResults.put("4", 1L); + expectedResults.put("5", 1L); + expectedResults.put("6", 1L); + checkValueInEachDoc(script, expectedResults, 6); + expectedResults.clear(); + + // check doc frequencies for term that does not exist + script = createScript("term = _index['float_payload_field']['non_existent_term']; if (term != null) {term.df()}"); + expectedResults.put("1", 0L); + expectedResults.put("2", 0L); + expectedResults.put("3", 0L); + expectedResults.put("4", 0L); + expectedResults.put("5", 0L); + expectedResults.put("6", 0L); + checkValueInEachDoc(script, expectedResults, 6); + expectedResults.clear(); + + // check doc frequencies for term that does not exist + script = createScript("term = _index['non_existent_field']['non_existent_term']; if (term != null) {term.tf()}"); + expectedResults.put("1", 0); + expectedResults.put("2", 0); + expectedResults.put("3", 0); + expectedResults.put("4", 0); + expectedResults.put("5", 0); + expectedResults.put("6", 0); + checkValueInEachDoc(script, expectedResults, 6); + expectedResults.clear(); + + // check total term frequencies for 'a' + script = createScript("term = _index['float_payload_field']['a']; if (term != null) {term.ttf()}"); + expectedResults.put("1", 4L); + expectedResults.put("2", 4L); + expectedResults.put("3", 4L); + expectedResults.put("4", 4L); + expectedResults.put("5", 4L); + expectedResults.put("6", 4L); + checkValueInEachDoc(script, expectedResults, 6); + expectedResults.clear(); + + // check float payload for 'b' + HashMap<String, List<Object>> expectedPayloadsArray = new HashMap<>(); + script = createPositionsArrayScript("float_payload_field", "b", INCLUDE_ALL, "payloadAsFloat(-1)"); + float missingValue = -1; + List<Object> payloadsFor1 = new ArrayList<>(); + payloadsFor1.add(2f); + payloadsFor1.add(missingValue); + expectedPayloadsArray.put("1", payloadsFor1); + List<Object> payloadsFor2 = new ArrayList<>(); + payloadsFor2.add(5f); + payloadsFor2.add(missingValue); + expectedPayloadsArray.put("3", payloadsFor2); + expectedPayloadsArray.put("6", new ArrayList<>()); + expectedPayloadsArray.put("5", new ArrayList<>()); + expectedPayloadsArray.put("4", new ArrayList<>()); + expectedPayloadsArray.put("2", new ArrayList<>()); + checkArrayValsInEachDoc(script, expectedPayloadsArray, 6); + + // check string payload for 'b' + expectedPayloadsArray.clear(); + payloadsFor1.clear(); + payloadsFor2.clear(); + script = createPositionsArrayScript("string_payload_field", "b", INCLUDE_ALL, "payloadAsString()"); + payloadsFor1.add("b"); + payloadsFor1.add(null); + expectedPayloadsArray.put("2", payloadsFor1); + payloadsFor2.add("a"); + payloadsFor2.add(null); + expectedPayloadsArray.put("4", payloadsFor2); + expectedPayloadsArray.put("6", new ArrayList<>()); + expectedPayloadsArray.put("5", new ArrayList<>()); + expectedPayloadsArray.put("3", new ArrayList<>()); + expectedPayloadsArray.put("1", new ArrayList<>()); + checkArrayValsInEachDoc(script, expectedPayloadsArray, 6); + + // check int payload for 'c' + expectedPayloadsArray.clear(); + payloadsFor1.clear(); + payloadsFor2.clear(); + script = createPositionsArrayScript("int_payload_field", "c", INCLUDE_ALL, "payloadAsInt(-1)"); + payloadsFor1 = new ArrayList<>(); + payloadsFor1.add(1); + expectedPayloadsArray.put("6", payloadsFor1); + expectedPayloadsArray.put("5", new ArrayList<>()); + expectedPayloadsArray.put("4", new ArrayList<>()); + expectedPayloadsArray.put("3", new ArrayList<>()); + expectedPayloadsArray.put("2", new ArrayList<>()); + expectedPayloadsArray.put("1", new ArrayList<>()); + checkArrayValsInEachDoc(script, expectedPayloadsArray, 6); + + } + + private void checkExceptions(Script script) { + try { + SearchResponse sr = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()).addScriptField("tvtest", script) + .execute().actionGet(); + assertThat(sr.getHits().hits().length, equalTo(0)); + ShardSearchFailure[] shardFails = sr.getShardFailures(); + for (ShardSearchFailure fail : shardFails) { + assertThat(fail.reason().indexOf("Cannot iterate twice! If you want to iterate more that once, add _CACHE explicitly."), + Matchers.greaterThan(-1)); + } + } catch (SearchPhaseExecutionException ex) { + assertThat( + "got " + ex.toString(), + ex.toString().indexOf("Cannot iterate twice! If you want to iterate more that once, add _CACHE explicitly."), + Matchers.greaterThan(-1)); + } + } + + private void checkValueInEachDocWithFunctionScore(Script fieldScript, Map<String, Object> expectedFieldVals, Script scoreScript, + Map<String, Object> expectedScore, int numExpectedDocs) { + SearchResponse sr = client().prepareSearch("test") + .setQuery(QueryBuilders.functionScoreQuery(ScoreFunctionBuilders.scriptFunction(scoreScript))) + .addScriptField("tvtest", fieldScript).execute().actionGet(); + assertHitCount(sr, numExpectedDocs); + for (SearchHit hit : sr.getHits().getHits()) { + Object result = hit.getFields().get("tvtest").getValues().get(0); + Object expectedResult = expectedFieldVals.get(hit.getId()); + assertThat("for doc " + hit.getId(), result, equalTo(expectedResult)); + assertThat("for doc " + hit.getId(), ((Float) expectedScore.get(hit.getId())).doubleValue(), + Matchers.closeTo(hit.score(), 1.e-4)); + } + } + + private void checkValueInEachDoc(Script script, Map<String, Object> expectedResults, int numExpectedDocs) { + SearchResponse sr = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()).addScriptField("tvtest", script) + .execute().actionGet(); + assertHitCount(sr, numExpectedDocs); + for (SearchHit hit : sr.getHits().getHits()) { + Object result = hit.getFields().get("tvtest").getValues().get(0); + Object expectedResult = expectedResults.get(hit.getId()); + assertThat("for doc " + hit.getId(), result, equalTo(expectedResult)); + } + } + + private void checkValueInEachDoc(int value, Script script, int numExpectedDocs) { + SearchResponse sr = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()).addScriptField("tvtest", script) + .execute().actionGet(); + assertHitCount(sr, numExpectedDocs); + for (SearchHit hit : sr.getHits().getHits()) { + Object result = hit.getFields().get("tvtest").getValues().get(0); + if (result instanceof Integer) { + assertThat(result, equalTo(value)); + } else if (result instanceof Long) { + assertThat(((Long) result).intValue(), equalTo(value)); + } else { + fail(); + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java index c7ee421e7e0..16a1c20792f 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java @@ -43,7 +43,9 @@ public class ScriptContextTests extends ESTestCase { .put("script." + PLUGIN_NAME + "_custom_globally_disabled_op", "false") .put("script.engine." + MockScriptEngine.NAME + ".inline." + PLUGIN_NAME + "_custom_exp_disabled_op", "false") .build(); - ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new MockScriptEngine())); + + MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap("1", script -> "1")); + ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(scriptEngine)); List<ScriptContext.Plugin> customContexts = Arrays.asList( new ScriptContext.Plugin(PLUGIN_NAME, "custom_op"), new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"), diff --git a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java index 6ae607e7b8e..1fc9ed8ff77 100644 --- a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java +++ b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java @@ -86,7 +86,7 @@ public class StoredScriptsIT extends ESIntegTestCase { @Override protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() { - return Collections.emptyMap(); + return Collections.singletonMap("1", script -> "1"); } } } diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RangeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java similarity index 83% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RangeTests.java rename to core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java index 9955d244fa5..d3e934d875f 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RangeTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java @@ -16,13 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.aggregations.bucket; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; -import org.elasticsearch.script.groovy.GroovyPlugin; +import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.range.Range; @@ -36,12 +37,15 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.script.ScriptService.ScriptType; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; -import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; import static org.elasticsearch.search.aggregations.AggregationBuilders.range; +import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; @@ -49,11 +53,8 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; import static org.hamcrest.core.IsNull.nullValue; -/** - * - */ @ESIntegTestCase.SuiteScopeTestCase -public class RangeTests extends ESIntegTestCase { +public class RangeIT extends ESIntegTestCase { private static final String SINGLE_VALUED_FIELD_NAME = "l_value"; private static final String MULTI_VALUED_FIELD_NAME = "l_values"; @@ -62,7 +63,30 @@ public class RangeTests extends ESIntegTestCase { @Override protected Collection<Class<? extends Plugin>> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); + return Collections.singleton(CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends AggregationTestScriptsPlugin { + + @Override + @SuppressWarnings("unchecked") + protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() { + Map<String, Function<Map<String, Object>, Object>> scripts = super.pluginScripts(); + + scripts.put("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", vars -> { + Map<?, ?> doc = (Map) vars.get("doc"); + ScriptDocValues.Longs value = (ScriptDocValues.Longs) doc.get(SINGLE_VALUED_FIELD_NAME); + return value.getValue(); + }); + + scripts.put("doc['" + MULTI_VALUED_FIELD_NAME + "'].values", vars -> { + Map<?, ?> doc = (Map) vars.get("doc"); + ScriptDocValues.Longs value = (ScriptDocValues.Longs) doc.get(MULTI_VALUED_FIELD_NAME); + return value.getValues(); + }); + + return scripts; + } } @Override @@ -94,10 +118,10 @@ public class RangeTests extends ESIntegTestCase { SearchResponse response = client().prepareSearch("idx") .addAggregation(terms("terms").field(MULTI_VALUED_FIELD_NAME).size(100) .collectMode(randomFrom(SubAggCollectionMode.values())).subAggregation( - range("range").field(SINGLE_VALUED_FIELD_NAME) - .addUnboundedTo(3) - .addRange(3, 6) - .addUnboundedFrom(6))) + range("range").field(SINGLE_VALUED_FIELD_NAME) + .addUnboundedTo(3) + .addRange(3, 6) + .addUnboundedFrom(6))) .execute().actionGet(); assertSearchResponse(response); @@ -112,7 +136,7 @@ public class RangeTests extends ESIntegTestCase { Range range = bucket.getAggregations().get("range"); List<? extends Bucket> buckets = range.getBuckets(); Range.Bucket rangeBucket = buckets.get(0); - assertThat((String) rangeBucket.getKey(), equalTo("*-3.0")); + assertThat(rangeBucket.getKey(), equalTo("*-3.0")); assertThat(rangeBucket.getKeyAsString(), equalTo("*-3.0")); assertThat(rangeBucket, notNullValue()); assertThat(rangeBucket.getFromAsString(), nullValue()); @@ -125,7 +149,7 @@ public class RangeTests extends ESIntegTestCase { assertThat(rangeBucket.getDocCount(), equalTo(0L)); } rangeBucket = buckets.get(1); - assertThat((String) rangeBucket.getKey(), equalTo("3.0-6.0")); + assertThat(rangeBucket.getKey(), equalTo("3.0-6.0")); assertThat(rangeBucket.getKeyAsString(), equalTo("3.0-6.0")); assertThat(rangeBucket, notNullValue()); assertThat(rangeBucket.getFromAsString(), equalTo("3.0")); @@ -138,7 +162,7 @@ public class RangeTests extends ESIntegTestCase { assertThat(rangeBucket.getDocCount(), equalTo(0L)); } rangeBucket = buckets.get(2); - assertThat((String) rangeBucket.getKey(), equalTo("6.0-*")); + assertThat(rangeBucket.getKey(), equalTo("6.0-*")); assertThat(rangeBucket.getKeyAsString(), equalTo("6.0-*")); assertThat(rangeBucket, notNullValue()); assertThat(rangeBucket.getFromAsString(), equalTo("6.0")); @@ -173,7 +197,7 @@ public class RangeTests extends ESIntegTestCase { Range.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("*-3.0")); + assertThat(bucket.getKey(), equalTo("*-3.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(3.0)); assertThat(bucket.getFromAsString(), nullValue()); @@ -182,7 +206,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(1); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("3.0-6.0")); + assertThat(bucket.getKey(), equalTo("3.0-6.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(3.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(6.0)); assertThat(bucket.getFromAsString(), equalTo("3.0")); @@ -191,7 +215,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(2); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("6.0-*")); + assertThat(bucket.getKey(), equalTo("6.0-*")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(6.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY)); assertThat(bucket.getFromAsString(), equalTo("6.0")); @@ -217,7 +241,7 @@ public class RangeTests extends ESIntegTestCase { Range.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("*-3")); + assertThat(bucket.getKey(), equalTo("*-3")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(3.0)); assertThat(bucket.getFromAsString(), nullValue()); @@ -226,7 +250,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(1); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("3-6")); + assertThat(bucket.getKey(), equalTo("3-6")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(3.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(6.0)); assertThat(bucket.getFromAsString(), equalTo("3")); @@ -235,7 +259,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(2); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("6-*")); + assertThat(bucket.getKey(), equalTo("6-*")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(6.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY)); assertThat(bucket.getFromAsString(), equalTo("6")); @@ -263,7 +287,7 @@ public class RangeTests extends ESIntegTestCase { Range.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("r1")); + assertThat(bucket.getKey(), equalTo("r1")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(3.0)); assertThat(bucket.getFromAsString(), nullValue()); @@ -272,7 +296,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(1); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("r2")); + assertThat(bucket.getKey(), equalTo("r2")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(3.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(6.0)); assertThat(bucket.getFromAsString(), equalTo("3.0")); @@ -281,7 +305,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(2); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("r3")); + assertThat(bucket.getKey(), equalTo("r3")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(6.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY)); assertThat(bucket.getFromAsString(), equalTo("6.0")); @@ -313,7 +337,7 @@ public class RangeTests extends ESIntegTestCase { Range.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("*-3.0")); + assertThat(bucket.getKey(), equalTo("*-3.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(3.0)); assertThat(bucket.getFromAsString(), nullValue()); @@ -322,13 +346,13 @@ public class RangeTests extends ESIntegTestCase { Sum sum = bucket.getAggregations().get("sum"); assertThat(sum, notNullValue()); assertThat(sum.getValue(), equalTo(3.0)); // 1 + 2 - assertThat((String) propertiesKeys[0], equalTo("*-3.0")); - assertThat((long) propertiesDocCounts[0], equalTo(2L)); - assertThat((double) propertiesCounts[0], equalTo(3.0)); + assertThat(propertiesKeys[0], equalTo("*-3.0")); + assertThat(propertiesDocCounts[0], equalTo(2L)); + assertThat(propertiesCounts[0], equalTo(3.0)); bucket = buckets.get(1); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("3.0-6.0")); + assertThat(bucket.getKey(), equalTo("3.0-6.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(3.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(6.0)); assertThat(bucket.getFromAsString(), equalTo("3.0")); @@ -337,13 +361,13 @@ public class RangeTests extends ESIntegTestCase { sum = bucket.getAggregations().get("sum"); assertThat(sum, notNullValue()); assertThat(sum.getValue(), equalTo(12.0)); // 3 + 4 + 5 - assertThat((String) propertiesKeys[1], equalTo("3.0-6.0")); - assertThat((long) propertiesDocCounts[1], equalTo(3L)); - assertThat((double) propertiesCounts[1], equalTo(12.0)); + assertThat(propertiesKeys[1], equalTo("3.0-6.0")); + assertThat(propertiesDocCounts[1], equalTo(3L)); + assertThat(propertiesCounts[1], equalTo(12.0)); bucket = buckets.get(2); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("6.0-*")); + assertThat(bucket.getKey(), equalTo("6.0-*")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(6.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY)); assertThat(bucket.getFromAsString(), equalTo("6.0")); @@ -356,17 +380,22 @@ public class RangeTests extends ESIntegTestCase { total += i + 1; } assertThat(sum.getValue(), equalTo((double) total)); - assertThat((String) propertiesKeys[2], equalTo("6.0-*")); - assertThat((long) propertiesDocCounts[2], equalTo(numDocs - 5L)); - assertThat((double) propertiesCounts[2], equalTo((double) total)); + assertThat(propertiesKeys[2], equalTo("6.0-*")); + assertThat(propertiesDocCounts[2], equalTo(numDocs - 5L)); + assertThat(propertiesCounts[2], equalTo((double) total)); } public void testSingleValuedFieldWithValueScript() throws Exception { SearchResponse response = client() .prepareSearch("idx") .addAggregation( - range("range").field(SINGLE_VALUED_FIELD_NAME).script(new Script("_value + 1")).addUnboundedTo(3).addRange(3, 6) - .addUnboundedFrom(6)).execute().actionGet(); + range("range") + .field(SINGLE_VALUED_FIELD_NAME) + .script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .addUnboundedTo(3) + .addRange(3, 6) + .addUnboundedFrom(6)) + .get(); assertSearchResponse(response); @@ -378,7 +407,7 @@ public class RangeTests extends ESIntegTestCase { Range.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("*-3.0")); + assertThat(bucket.getKey(), equalTo("*-3.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(3.0)); assertThat(bucket.getFromAsString(), nullValue()); @@ -387,7 +416,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(1); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("3.0-6.0")); + assertThat(bucket.getKey(), equalTo("3.0-6.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(3.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(6.0)); assertThat(bucket.getFromAsString(), equalTo("3.0")); @@ -396,7 +425,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(2); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("6.0-*")); + assertThat(bucket.getKey(), equalTo("6.0-*")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(6.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY)); assertThat(bucket.getFromAsString(), equalTo("6.0")); @@ -437,7 +466,7 @@ public class RangeTests extends ESIntegTestCase { Range.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("*-3.0")); + assertThat(bucket.getKey(), equalTo("*-3.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(3.0)); assertThat(bucket.getFromAsString(), nullValue()); @@ -446,7 +475,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(1); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("3.0-6.0")); + assertThat(bucket.getKey(), equalTo("3.0-6.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(3.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(6.0)); assertThat(bucket.getFromAsString(), equalTo("3.0")); @@ -455,7 +484,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(2); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("6.0-*")); + assertThat(bucket.getKey(), equalTo("6.0-*")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(6.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY)); assertThat(bucket.getFromAsString(), equalTo("6.0")); @@ -480,8 +509,13 @@ public class RangeTests extends ESIntegTestCase { SearchResponse response = client() .prepareSearch("idx") .addAggregation( - range("range").field(MULTI_VALUED_FIELD_NAME).script(new Script("_value + 1")).addUnboundedTo(3).addRange(3, 6) - .addUnboundedFrom(6)).execute().actionGet(); + range("range") + .field(MULTI_VALUED_FIELD_NAME) + .script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .addUnboundedTo(3) + .addRange(3, 6) + .addUnboundedFrom(6)) + .get(); assertSearchResponse(response); @@ -494,7 +528,7 @@ public class RangeTests extends ESIntegTestCase { Range.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("*-3.0")); + assertThat(bucket.getKey(), equalTo("*-3.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(3.0)); assertThat(bucket.getFromAsString(), nullValue()); @@ -503,7 +537,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(1); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("3.0-6.0")); + assertThat(bucket.getKey(), equalTo("3.0-6.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(3.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(6.0)); assertThat(bucket.getFromAsString(), equalTo("3.0")); @@ -512,7 +546,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(2); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("6.0-*")); + assertThat(bucket.getKey(), equalTo("6.0-*")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(6.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY)); assertThat(bucket.getFromAsString(), equalTo("6.0")); @@ -538,11 +572,16 @@ public class RangeTests extends ESIntegTestCase { */ public void testScriptSingleValue() throws Exception { + Script script = new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null); SearchResponse response = client() .prepareSearch("idx") .addAggregation( - range("range").script(new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value")).addUnboundedTo(3).addRange(3, 6) - .addUnboundedFrom(6)).execute().actionGet(); + range("range") + .script(script) + .addUnboundedTo(3) + .addRange(3, 6) + .addUnboundedFrom(6)) + .get(); assertSearchResponse(response); @@ -555,7 +594,7 @@ public class RangeTests extends ESIntegTestCase { Range.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("*-3.0")); + assertThat(bucket.getKey(), equalTo("*-3.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(3.0)); assertThat(bucket.getFromAsString(), nullValue()); @@ -564,7 +603,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(1); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("3.0-6.0")); + assertThat(bucket.getKey(), equalTo("3.0-6.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(3.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(6.0)); assertThat(bucket.getFromAsString(), equalTo("3.0")); @@ -573,7 +612,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(2); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("6.0-*")); + assertThat(bucket.getKey(), equalTo("6.0-*")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(6.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY)); assertThat(bucket.getFromAsString(), equalTo("6.0")); @@ -600,7 +639,7 @@ public class RangeTests extends ESIntegTestCase { Range.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("*--1.0")); + assertThat(bucket.getKey(), equalTo("*--1.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(-1.0)); assertThat(bucket.getFromAsString(), nullValue()); @@ -609,7 +648,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(1); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("1000.0-*")); + assertThat(bucket.getKey(), equalTo("1000.0-*")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(1000d)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY)); assertThat(bucket.getFromAsString(), equalTo("1000.0")); @@ -618,11 +657,17 @@ public class RangeTests extends ESIntegTestCase { } public void testScriptMultiValued() throws Exception { + Script script = new Script("doc['" + MULTI_VALUED_FIELD_NAME + "'].values", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse response = client() .prepareSearch("idx") .addAggregation( - range("range").script(new Script("doc['" + MULTI_VALUED_FIELD_NAME + "'].values")).addUnboundedTo(3).addRange(3, 6) - .addUnboundedFrom(6)).execute().actionGet(); + range("range") + .script(script) + .addUnboundedTo(3) + .addRange(3, 6) + .addUnboundedFrom(6)) + .get(); assertSearchResponse(response); @@ -635,7 +680,7 @@ public class RangeTests extends ESIntegTestCase { Range.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("*-3.0")); + assertThat(bucket.getKey(), equalTo("*-3.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(3.0)); assertThat(bucket.getFromAsString(), nullValue()); @@ -644,7 +689,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(1); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("3.0-6.0")); + assertThat(bucket.getKey(), equalTo("3.0-6.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(3.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(6.0)); assertThat(bucket.getFromAsString(), equalTo("3.0")); @@ -653,7 +698,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(2); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("6.0-*")); + assertThat(bucket.getKey(), equalTo("6.0-*")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(6.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY)); assertThat(bucket.getFromAsString(), equalTo("6.0")); @@ -698,7 +743,7 @@ public class RangeTests extends ESIntegTestCase { Range.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("*-3.0")); + assertThat(bucket.getKey(), equalTo("*-3.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(3.0)); assertThat(bucket.getFromAsString(), nullValue()); @@ -707,7 +752,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(1); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("3.0-6.0")); + assertThat(bucket.getKey(), equalTo("3.0-6.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(3.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(6.0)); assertThat(bucket.getFromAsString(), equalTo("3.0")); @@ -716,7 +761,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(2); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("6.0-*")); + assertThat(bucket.getKey(), equalTo("6.0-*")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(6.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY)); assertThat(bucket.getFromAsString(), equalTo("6.0")); @@ -746,7 +791,7 @@ public class RangeTests extends ESIntegTestCase { Range.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("*-3.0")); + assertThat(bucket.getKey(), equalTo("*-3.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(3.0)); assertThat(bucket.getFromAsString(), nullValue()); @@ -755,7 +800,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(1); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("3.0-6.0")); + assertThat(bucket.getKey(), equalTo("3.0-6.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(3.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(6.0)); assertThat(bucket.getFromAsString(), equalTo("3.0")); @@ -764,7 +809,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(2); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("6.0-*")); + assertThat(bucket.getKey(), equalTo("6.0-*")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(6.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY)); assertThat(bucket.getFromAsString(), equalTo("6.0")); @@ -793,7 +838,7 @@ public class RangeTests extends ESIntegTestCase { Range.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("*-5.0")); + assertThat(bucket.getKey(), equalTo("*-5.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(5.0)); assertThat(bucket.getFromAsString(), nullValue()); @@ -802,7 +847,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(1); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("3.0-6.0")); + assertThat(bucket.getKey(), equalTo("3.0-6.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(3.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(6.0)); assertThat(bucket.getFromAsString(), equalTo("3.0")); @@ -811,7 +856,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(2); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("4.0-5.0")); + assertThat(bucket.getKey(), equalTo("4.0-5.0")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(4.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(5.0)); assertThat(bucket.getFromAsString(), equalTo("4.0")); @@ -820,7 +865,7 @@ public class RangeTests extends ESIntegTestCase { bucket = buckets.get(3); assertThat(bucket, notNullValue()); - assertThat((String) bucket.getKey(), equalTo("4.0-*")); + assertThat(bucket.getKey(), equalTo("4.0-*")); assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(4.0)); assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY)); assertThat(bucket.getFromAsString(), equalTo("4.0")); @@ -831,9 +876,16 @@ public class RangeTests extends ESIntegTestCase { public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) - .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(1L).minDocCount(0) - .subAggregation(range("range").field(SINGLE_VALUED_FIELD_NAME).addRange("0-2", 0.0, 2.0))) - .execute().actionGet(); + .addAggregation( + histogram("histo") + .field(SINGLE_VALUED_FIELD_NAME) + .interval(1L) + .minDocCount(0) + .subAggregation( + range("range") + .field(SINGLE_VALUED_FIELD_NAME) + .addRange("0-2", 0.0, 2.0))) + .get(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L)); Histogram histo = searchResponse.getAggregations().get("histo"); @@ -843,11 +895,11 @@ public class RangeTests extends ESIntegTestCase { Range range = bucket.getAggregations().get("range"); // TODO: use diamond once JI-9019884 is fixed - List<Range.Bucket> buckets = new ArrayList<Range.Bucket>(range.getBuckets()); + List<Range.Bucket> buckets = new ArrayList<>(range.getBuckets()); assertThat(range, Matchers.notNullValue()); assertThat(range.getName(), equalTo("range")); assertThat(buckets.size(), is(1)); - assertThat((String) buckets.get(0).getKey(), equalTo("0-2")); + assertThat(buckets.get(0).getKey(), equalTo("0-2")); assertThat(((Number) buckets.get(0).getFrom()).doubleValue(), equalTo(0.0)); assertThat(((Number) buckets.get(0).getTo()).doubleValue(), equalTo(2.0)); assertThat(buckets.get(0).getFromAsString(), equalTo("0.0")); diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptedMetricTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java similarity index 68% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptedMetricTests.java rename to core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java index 713ef12e1a7..e1800b2f9f1 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptedMetricTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java @@ -17,18 +17,18 @@ * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.groovy.GroovyPlugin; -import org.elasticsearch.script.groovy.GroovyScriptEngineService; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.bucket.global.Global; @@ -39,12 +39,17 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; @@ -64,13 +69,130 @@ import static org.hamcrest.Matchers.sameInstance; @ClusterScope(scope = Scope.SUITE) @ESIntegTestCase.SuiteScopeTestCase -public class ScriptedMetricTests extends ESIntegTestCase { +public class ScriptedMetricIT extends ESIntegTestCase { private static long numDocs; @Override protected Collection<Class<? extends Plugin>> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); + return Collections.singleton(CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends MockScriptPlugin { + + @Override + @SuppressWarnings("unchecked") + protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() { + Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>(); + + scripts.put("_agg['count'] = 1", vars -> + aggScript(vars, agg -> ((Map<String, Object>) agg).put("count", 1))); + + scripts.put("_agg.add(1)", vars -> + aggScript(vars, agg -> ((List) agg).add(1))); + + scripts.put("vars.multiplier = 3", vars -> + ((Map<String, Object>) vars.get("vars")).put("multiplier", 3)); + + scripts.put("_agg.add(vars.multiplier)", vars -> + aggScript(vars, agg -> ((List) agg).add(XContentMapValues.extractValue("vars.multiplier", vars)))); + + // Equivalent to: + // + // newaggregation = []; + // sum = 0; + // + // for (a in _agg) { + // sum += a + // }; + // + // newaggregation.add(sum); + // return newaggregation" + // + scripts.put("sum agg values as a new aggregation", vars -> { + List newAggregation = new ArrayList(); + List<?> agg = (List<?>) vars.get("_agg"); + + if (agg != null) { + Integer sum = 0; + for (Object a : (List) agg) { + sum += ((Number) a).intValue(); + } + newAggregation.add(sum); + } + return newAggregation; + }); + + // Equivalent to: + // + // newaggregation = []; + // sum = 0; + // + // for (aggregation in _aggs) { + // for (a in aggregation) { + // sum += a + // } + // }; + // + // newaggregation.add(sum); + // return newaggregation" + // + scripts.put("sum aggs of agg values as a new aggregation", vars -> { + List newAggregation = new ArrayList(); + Integer sum = 0; + + List<?> aggs = (List<?>) vars.get("_aggs"); + for (Object aggregation : (List) aggs) { + if (aggregation != null) { + for (Object a : (List) aggregation) { + sum += ((Number) a).intValue(); + } + } + } + newAggregation.add(sum); + return newAggregation; + }); + + // Equivalent to: + // + // newaggregation = []; + // sum = 0; + // + // for (aggregation in _aggs) { + // for (a in aggregation) { + // sum += a + // } + // }; + // + // newaggregation.add(sum * multiplier); + // return newaggregation" + // + scripts.put("multiplied sum aggs of agg values as a new aggregation", vars -> { + Integer multiplier = (Integer) vars.get("multiplier"); + List newAggregation = new ArrayList(); + Integer sum = 0; + + List<?> aggs = (List<?>) vars.get("_aggs"); + for (Object aggregation : (List) aggs) { + if (aggregation != null) { + for (Object a : (List) aggregation) { + sum += ((Number) a).intValue(); + } + } + } + newAggregation.add(sum * multiplier); + return newAggregation; + }); + + return scripts; + } + + @SuppressWarnings("unchecked") + static <T> Object aggScript(Map<String, Object> vars, Consumer<T> fn) { + T agg = (T) vars.get("_agg"); + fn.accept(agg); + return agg; + } } @Override @@ -83,7 +205,7 @@ public class ScriptedMetricTests extends ESIntegTestCase { for (int i = 0; i < numDocs; i++) { builders.add(client().prepareIndex("idx", "type", "" + i).setSource( jsonBuilder().startObject().field("value", randomAsciiOfLengthBetween(5, 15)) - .field("l_value", i).endObject())); + .field("l_value", i).endObject())); } indexRandom(true, builders); @@ -102,25 +224,28 @@ public class ScriptedMetricTests extends ESIntegTestCase { jsonBuilder().startObject().field("value", i * 2).endObject())); } + // When using the MockScriptPlugin we can map Stored scripts to inline scripts: + // the id of the stored script is used in test method while the source of the stored script + // must match a predefined script from CustomScriptPlugin.pluginScripts() method assertAcked(client().admin().cluster().preparePutStoredScript() - .setScriptLang(GroovyScriptEngineService.NAME) - .setId("initScript_indexed") + .setScriptLang(CustomScriptPlugin.NAME) + .setId("initScript_stored") .setSource(new BytesArray("{\"script\":\"vars.multiplier = 3\"}"))); assertAcked(client().admin().cluster().preparePutStoredScript() - .setScriptLang(GroovyScriptEngineService.NAME) - .setId("mapScript_indexed") + .setScriptLang(CustomScriptPlugin.NAME) + .setId("mapScript_stored") .setSource(new BytesArray("{\"script\":\"_agg.add(vars.multiplier)\"}"))); assertAcked(client().admin().cluster().preparePutStoredScript() - .setScriptLang(GroovyScriptEngineService.NAME) - .setId("combineScript_indexed") - .setSource(new BytesArray("{\"script\":\"newaggregation = []; sum = 0;for (a in _agg) { sum += a}; newaggregation.add(sum); return newaggregation\"}"))); + .setScriptLang(CustomScriptPlugin.NAME) + .setId("combineScript_stored") + .setSource(new BytesArray("{\"script\":\"sum agg values as a new aggregation\"}"))); assertAcked(client().admin().cluster().preparePutStoredScript() - .setScriptLang(GroovyScriptEngineService.NAME) - .setId("reduceScript_indexed") - .setSource(new BytesArray("{\"script\":\"newaggregation = []; sum = 0;for (agg in _aggs) { for (a in agg) { sum += a} }; newaggregation.add(sum); return newaggregation\"}"))); + .setScriptLang(CustomScriptPlugin.NAME) + .setId("reduceScript_stored") + .setSource(new BytesArray("{\"script\":\"sum aggs of agg values as a new aggregation\"}"))); indexRandom(true, builders); ensureSearchable(); @@ -128,16 +253,36 @@ public class ScriptedMetricTests extends ESIntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { - Settings settings = Settings.builder() + Path config = createTempDir().resolve("config"); + Path scripts = config.resolve("scripts"); + + try { + Files.createDirectories(scripts); + + // When using the MockScriptPlugin we can map File scripts to inline scripts: + // the name of the file script is used in test method while the source of the file script + // must match a predefined script from CustomScriptPlugin.pluginScripts() method + Files.write(scripts.resolve("init_script.mockscript"), "vars.multiplier = 3".getBytes("UTF-8")); + Files.write(scripts.resolve("map_script.mockscript"), "_agg.add(vars.multiplier)".getBytes("UTF-8")); + Files.write(scripts.resolve("combine_script.mockscript"), "sum agg values as a new aggregation".getBytes("UTF-8")); + Files.write(scripts.resolve("reduce_script.mockscript"), "sum aggs of agg values as a new aggregation".getBytes("UTF-8")); + } catch (IOException e) { + throw new RuntimeException("failed to create scripts"); + } + + return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) - .put(Environment.PATH_CONF_SETTING.getKey(), getDataPath("/org/elasticsearch/messy/tests/conf")) + .put(Environment.PATH_CONF_SETTING.getKey(), config) .build(); - return settings; } public void testMap() { - SearchResponse response = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(scriptedMetric("scripted").mapScript(new Script("_agg['count'] = 1"))).execute().actionGet(); + Script mapScript = new Script("_agg['count'] = 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + + SearchResponse response = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(scriptedMetric("scripted").mapScript(mapScript)) + .get(); assertSearchResponse(response); assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); @@ -159,7 +304,7 @@ public class ScriptedMetricTests extends ESIntegTestCase { if (map.size() == 1) { assertThat(map.get("count"), notNullValue()); assertThat(map.get("count"), instanceOf(Number.class)); - assertThat((Number) map.get("count"), equalTo((Number) 1)); + assertThat(map.get("count"), equalTo((Number) 1)); numShardsRun++; } } @@ -172,8 +317,12 @@ public class ScriptedMetricTests extends ESIntegTestCase { Map<String, Object> params = new HashMap<>(); params.put("_agg", new ArrayList<>()); - SearchResponse response = client().prepareSearch("idx").setQuery(matchAllQuery()) - .addAggregation(scriptedMetric("scripted").params(params).mapScript(new Script("_agg.add(1)"))).execute().actionGet(); + Script mapScript = new Script("_agg.add(1)", ScriptType.INLINE, CustomScriptPlugin.NAME, params); + + SearchResponse response = client().prepareSearch("idx") + .setQuery(matchAllQuery()) + .addAggregation(scriptedMetric("scripted").params(params).mapScript(mapScript)) + .get(); assertSearchResponse(response); assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); @@ -205,6 +354,7 @@ public class ScriptedMetricTests extends ESIntegTestCase { public void testInitMapWithParams() { Map<String, Object> varsMap = new HashMap<>(); varsMap.put("multiplier", 1); + Map<String, Object> params = new HashMap<>(); params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); @@ -213,8 +363,11 @@ public class ScriptedMetricTests extends ESIntegTestCase { .prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation( - scriptedMetric("scripted").params(params).initScript(new Script("vars.multiplier = 3")) - .mapScript(new Script("_agg.add(vars.multiplier)"))).execute().actionGet(); + scriptedMetric("scripted") + .params(params) + .initScript(new Script("vars.multiplier = 3", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .mapScript(new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null))) + .get(); assertSearchResponse(response); assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); @@ -246,20 +399,22 @@ public class ScriptedMetricTests extends ESIntegTestCase { public void testMapCombineWithParams() { Map<String, Object> varsMap = new HashMap<>(); varsMap.put("multiplier", 1); + Map<String, Object> params = new HashMap<>(); params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); + Script mapScript = new Script("_agg.add(1)", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script combineScript = new Script("sum agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse response = client() .prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation( scriptedMetric("scripted") .params(params) - .mapScript(new Script("_agg.add(1)")) - .combineScript( - new Script( - "newaggregation = []; sum = 0;for (a in _agg) { sum += a}; newaggregation.add(sum); return newaggregation"))) + .mapScript(mapScript) + .combineScript(combineScript)) .execute().actionGet(); assertSearchResponse(response); assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); @@ -295,22 +450,25 @@ public class ScriptedMetricTests extends ESIntegTestCase { public void testInitMapCombineWithParams() { Map<String, Object> varsMap = new HashMap<>(); varsMap.put("multiplier", 1); + Map<String, Object> params = new HashMap<>(); params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); + Script initScript = new Script("vars.multiplier = 3", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script combineScript = new Script("sum agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse response = client() .prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation( scriptedMetric("scripted") .params(params) - .initScript(new Script("vars.multiplier = 3")) - .mapScript(new Script("_agg.add(vars.multiplier)")) - .combineScript( - new Script( - "newaggregation = []; sum = 0;for (a in _agg) { sum += a}; newaggregation.add(sum); return newaggregation"))) - .execute().actionGet(); + .initScript(initScript) + .mapScript(mapScript) + .combineScript(combineScript)) + .get(); assertSearchResponse(response); assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); @@ -345,25 +503,27 @@ public class ScriptedMetricTests extends ESIntegTestCase { public void testInitMapCombineReduceWithParams() { Map<String, Object> varsMap = new HashMap<>(); varsMap.put("multiplier", 1); + Map<String, Object> params = new HashMap<>(); params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); + Script initScript = new Script("vars.multiplier = 3", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script combineScript = new Script("sum agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script reduceScript = new Script("sum aggs of agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse response = client() .prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation( scriptedMetric("scripted") .params(params) - .initScript(new Script("vars.multiplier = 3")) - .mapScript(new Script("_agg.add(vars.multiplier)")) - .combineScript( - new Script( - "newaggregation = []; sum = 0;for (a in _agg) { sum += a}; newaggregation.add(sum); return newaggregation")) - .reduceScript( - new Script( - "newaggregation = []; sum = 0;for (aggregation in _aggs) { for (a in aggregation) { sum += a} }; newaggregation.add(sum); return newaggregation"))) - .execute().actionGet(); + .initScript(initScript) + .mapScript(mapScript) + .combineScript(combineScript) + .reduceScript(reduceScript)) + .get(); assertSearchResponse(response); assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); @@ -386,9 +546,16 @@ public class ScriptedMetricTests extends ESIntegTestCase { public void testInitMapCombineReduceGetProperty() throws Exception { Map<String, Object> varsMap = new HashMap<>(); varsMap.put("multiplier", 1); + Map<String, Object> params = new HashMap<>(); params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); + + Script initScript = new Script("vars.multiplier = 3", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script combineScript = new Script("sum agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script reduceScript = new Script("sum aggs of agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse searchResponse = client() .prepareSearch("idx") .setQuery(matchAllQuery()) @@ -397,15 +564,11 @@ public class ScriptedMetricTests extends ESIntegTestCase { .subAggregation( scriptedMetric("scripted") .params(params) - .initScript(new Script("vars.multiplier = 3")) - .mapScript(new Script("_agg.add(vars.multiplier)")) - .combineScript( - new Script( - "newaggregation = []; sum = 0;for (a in _agg) { sum += a}; newaggregation.add(sum); return newaggregation")) - .reduceScript( - new Script( - "newaggregation = []; sum = 0;for (aggregation in _aggs) { for (a in aggregation) { sum += a} }; newaggregation.add(sum); return newaggregation")))) - .execute().actionGet(); + .initScript(initScript) + .mapScript(mapScript) + .combineScript(combineScript) + .reduceScript(reduceScript))) + .get(); assertSearchResponse(searchResponse); assertThat(searchResponse.getHits().getTotalHits(), equalTo(numDocs)); @@ -437,24 +600,25 @@ public class ScriptedMetricTests extends ESIntegTestCase { public void testMapCombineReduceWithParams() { Map<String, Object> varsMap = new HashMap<>(); varsMap.put("multiplier", 1); + Map<String, Object> params = new HashMap<>(); params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); + Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script combineScript = new Script("sum agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script reduceScript = new Script("sum aggs of agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse response = client() .prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation( scriptedMetric("scripted") .params(params) - .mapScript(new Script("_agg.add(vars.multiplier)")) - .combineScript( - new Script( - "newaggregation = []; sum = 0;for (a in _agg) { sum += a}; newaggregation.add(sum); return newaggregation")) - .reduceScript( - new Script( - "newaggregation = []; sum = 0;for (aggregation in _aggs) { for (a in aggregation) { sum += a} }; newaggregation.add(sum); return newaggregation"))) - .execute().actionGet(); + .mapScript(mapScript) + .combineScript(combineScript) + .reduceScript(reduceScript)) + .get(); assertSearchResponse(response); assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); @@ -476,22 +640,25 @@ public class ScriptedMetricTests extends ESIntegTestCase { public void testInitMapReduceWithParams() { Map<String, Object> varsMap = new HashMap<>(); varsMap.put("multiplier", 1); + Map<String, Object> params = new HashMap<>(); params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); + Script initScript = new Script("vars.multiplier = 3", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script reduceScript = new Script("sum aggs of agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse response = client() .prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation( scriptedMetric("scripted") .params(params) - .initScript(new Script("vars.multiplier = 3")) - .mapScript(new Script("_agg.add(vars.multiplier)")) - .reduceScript( - new Script( - "newaggregation = []; sum = 0;for (aggregation in _aggs) { for (a in aggregation) { sum += a} }; newaggregation.add(sum); return newaggregation"))) - .execute().actionGet(); + .initScript(initScript) + .mapScript(mapScript) + .reduceScript(reduceScript)) + .get(); assertSearchResponse(response); assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); @@ -517,17 +684,18 @@ public class ScriptedMetricTests extends ESIntegTestCase { params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); + Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script reduceScript = new Script("sum aggs of agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse response = client() .prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation( scriptedMetric("scripted") .params(params) - .mapScript(new Script("_agg.add(vars.multiplier)")) - .reduceScript( - new Script( - "newaggregation = []; sum = 0;for (aggregation in _aggs) { for (a in aggregation) { sum += a} }; newaggregation.add(sum); return newaggregation"))) - .execute().actionGet(); + .mapScript(mapScript) + .reduceScript(reduceScript)) + .get(); assertSearchResponse(response); assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); @@ -549,27 +717,30 @@ public class ScriptedMetricTests extends ESIntegTestCase { public void testInitMapCombineReduceWithParamsAndReduceParams() { Map<String, Object> varsMap = new HashMap<>(); varsMap.put("multiplier", 1); + Map<String, Object> params = new HashMap<>(); params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); + Map<String, Object> reduceParams = new HashMap<>(); reduceParams.put("multiplier", 4); + Script initScript = new Script("vars.multiplier = 3", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script combineScript = new Script("sum agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script reduceScript = new Script("multiplied sum aggs of agg values as a new aggregation", ScriptType.INLINE, + CustomScriptPlugin.NAME, reduceParams); + SearchResponse response = client() .prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation( scriptedMetric("scripted") .params(params) - .initScript(new Script("vars.multiplier = 3")) - .mapScript(new Script("_agg.add(vars.multiplier)")) - .combineScript( - new Script( - "newaggregation = []; sum = 0;for (a in _agg) { sum += a}; newaggregation.add(sum); return newaggregation")) - .reduceScript( - new Script( - "newaggregation = []; sum = 0;for (aggregation in _aggs) { for (a in aggregation) { sum += a} }; newaggregation.add(sum * multiplier); return newaggregation", - ScriptType.INLINE, null, reduceParams))) + .initScript(initScript) + .mapScript(mapScript) + .combineScript(combineScript) + .reduceScript(reduceScript)) .execute().actionGet(); assertSearchResponse(response); assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); @@ -589,9 +760,10 @@ public class ScriptedMetricTests extends ESIntegTestCase { assertThat(((Number) object).longValue(), equalTo(numDocs * 12)); } - public void testInitMapCombineReduceWithParamsIndexed() { + public void testInitMapCombineReduceWithParamsStored() { Map<String, Object> varsMap = new HashMap<>(); varsMap.put("multiplier", 1); + Map<String, Object> params = new HashMap<>(); params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); @@ -600,11 +772,13 @@ public class ScriptedMetricTests extends ESIntegTestCase { .prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation( - scriptedMetric("scripted").params(params) - .initScript(new Script("initScript_indexed", ScriptType.STORED, null, null)) - .mapScript(new Script("mapScript_indexed", ScriptType.STORED, null, null)) - .combineScript(new Script("combineScript_indexed", ScriptType.STORED, null, null)) - .reduceScript(new Script("reduceScript_indexed", ScriptType.STORED, null, null))).execute().actionGet(); + scriptedMetric("scripted") + .params(params) + .initScript(new Script("initScript_stored", ScriptType.STORED, CustomScriptPlugin.NAME, null)) + .mapScript(new Script("mapScript_stored", ScriptType.STORED, CustomScriptPlugin.NAME, null)) + .combineScript(new Script("combineScript_stored", ScriptType.STORED, CustomScriptPlugin.NAME, null)) + .reduceScript(new Script("reduceScript_stored", ScriptType.STORED, CustomScriptPlugin.NAME, null))) + .get(); assertSearchResponse(response); assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); @@ -634,10 +808,13 @@ public class ScriptedMetricTests extends ESIntegTestCase { .prepareSearch("idx") .setQuery(matchAllQuery()) .addAggregation( - scriptedMetric("scripted").params(params).initScript(new Script("init_script", ScriptType.FILE, null, null)) - .mapScript(new Script("map_script", ScriptType.FILE, null, null)) - .combineScript(new Script("combine_script", ScriptType.FILE, null, null)) - .reduceScript(new Script("reduce_script", ScriptType.FILE, null, null))).execute().actionGet(); + scriptedMetric("scripted") + .params(params) + .initScript(new Script("init_script", ScriptType.FILE, CustomScriptPlugin.NAME, null)) + .mapScript(new Script("map_script", ScriptType.FILE, CustomScriptPlugin.NAME, null)) + .combineScript(new Script("combine_script", ScriptType.FILE, CustomScriptPlugin.NAME, null)) + .reduceScript(new Script("reduce_script", ScriptType.FILE, CustomScriptPlugin.NAME, null))) + .get(); assertSearchResponse(response); assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); @@ -659,10 +836,16 @@ public class ScriptedMetricTests extends ESIntegTestCase { public void testInitMapCombineReduceWithParamsAsSubAgg() { Map<String, Object> varsMap = new HashMap<>(); varsMap.put("multiplier", 1); + Map<String, Object> params = new HashMap<>(); params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); + Script initScript = new Script("vars.multiplier = 3", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script combineScript = new Script("sum agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script reduceScript = new Script("sum aggs of agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse response = client() .prepareSearch("idx") .setQuery(matchAllQuery()).setSize(1000) @@ -673,15 +856,11 @@ public class ScriptedMetricTests extends ESIntegTestCase { .subAggregation( scriptedMetric("scripted") .params(params) - .initScript(new Script("vars.multiplier = 3")) - .mapScript(new Script("_agg.add(vars.multiplier)")) - .combineScript( - new Script( - "newaggregation = []; sum = 0;for (a in _agg) { sum += a}; newaggregation.add(sum); return newaggregation")) - .reduceScript( - new Script( - "newaggregation = []; sum = 0;for (aggregation in _aggs) { for (a in aggregation) { sum += a} }; newaggregation.add(sum); return newaggregation")))) - .execute().actionGet(); + .initScript(initScript) + .mapScript(mapScript) + .combineScript(combineScript) + .reduceScript(reduceScript))) + .get(); assertSearchResponse(response); assertThat(response.getHits().getTotalHits(), equalTo(numDocs)); Aggregation aggregation = response.getAggregations().get("histo"); @@ -716,25 +895,27 @@ public class ScriptedMetricTests extends ESIntegTestCase { public void testEmptyAggregation() throws Exception { Map<String, Object> varsMap = new HashMap<>(); varsMap.put("multiplier", 1); + Map<String, Object> params = new HashMap<>(); params.put("_agg", new ArrayList<>()); params.put("vars", varsMap); + Script initScript = new Script("vars.multiplier = 3", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script mapScript = new Script("_agg.add(vars.multiplier)", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script combineScript = new Script("sum agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + Script reduceScript = new Script("sum aggs of agg values as a new aggregation", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") .setQuery(matchAllQuery()) .addAggregation(histogram("histo").field("value").interval(1L).minDocCount(0) .subAggregation( - scriptedMetric("scripted") - .params(params) - .initScript(new Script("vars.multiplier = 3")) - .mapScript(new Script("_agg.add(vars.multiplier)")) - .combineScript( - new Script( - "newaggregation = []; sum = 0;for (a in _agg) { sum += a}; newaggregation.add(sum); return newaggregation")) - .reduceScript( - new Script( - "newaggregation = []; sum = 0;for (aggregation in _aggs) { for (a in aggregation) { sum += a} }; newaggregation.add(sum); return newaggregation")))) - .execute().actionGet(); + scriptedMetric("scripted") + .params(params) + .initScript(initScript) + .mapScript(mapScript) + .combineScript(combineScript) + .reduceScript(reduceScript))) + .get(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L)); Histogram histo = searchResponse.getAggregations().get("histo"); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java index bfd4ed8ada9..e2b91b59fe3 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java @@ -101,7 +101,7 @@ public class TopHitsIT extends ESIntegTestCase { public static class CustomScriptPlugin extends MockScriptPlugin { @Override protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() { - return Collections.emptyMap(); + return Collections.singletonMap("5", script -> "5"); } } diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java similarity index 69% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java rename to core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index 03b85d57e0e..a9f73935504 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java +++ b/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -17,28 +17,30 @@ * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.fields; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.Priority; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHitField; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.lookup.FieldLookup; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.joda.time.DateTime; @@ -49,11 +51,13 @@ import java.util.Arrays; import java.util.Base64; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.function.Function; import static java.util.Collections.singleton; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; @@ -72,13 +76,82 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -/** - * - */ -public class SearchFieldsTests extends ESIntegTestCase { +public class SearchFieldsIT extends ESIntegTestCase { + @Override protected Collection<Class<? extends Plugin>> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); + return pluginList(CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends MockScriptPlugin { + + @Override + @SuppressWarnings("unchecked") + protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() { + Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>(); + + scripts.put("doc['num1'].value", vars -> { + Map<?, ?> doc = (Map) vars.get("doc"); + ScriptDocValues.Doubles num1 = (ScriptDocValues.Doubles) doc.get("num1"); + return num1.getValue(); + }); + + scripts.put("doc['num1'].value * factor", vars -> { + Map<?, ?> doc = (Map) vars.get("doc"); + ScriptDocValues.Doubles num1 = (ScriptDocValues.Doubles) doc.get("num1"); + Double factor = (Double) vars.get("factor"); + return num1.getValue() * factor; + }); + + scripts.put("doc['date'].date.millis", vars -> { + Map<?, ?> doc = (Map) vars.get("doc"); + ScriptDocValues.Longs date = (ScriptDocValues.Longs) doc.get("date"); + return date.getDate().getMillis(); + }); + + scripts.put("_fields['num1'].value", vars -> fieldsScript(vars, "num1")); + scripts.put("_fields._uid.value", vars -> fieldsScript(vars, "_uid")); + scripts.put("_fields._id.value", vars -> fieldsScript(vars, "_id")); + scripts.put("_fields._type.value", vars -> fieldsScript(vars, "_type")); + + scripts.put("_source.obj1", vars -> sourceScript(vars, "obj1")); + scripts.put("_source.obj1.test", vars -> sourceScript(vars, "obj1.test")); + scripts.put("_source.obj1.test", vars -> sourceScript(vars, "obj1.test")); + scripts.put("_source.obj2", vars -> sourceScript(vars, "obj2")); + scripts.put("_source.obj2.arr2", vars -> sourceScript(vars, "obj2.arr2")); + scripts.put("_source.arr3", vars -> sourceScript(vars, "arr3")); + + scripts.put("return null", vars -> null); + + scripts.put("doc['l'].values", vars -> docScript(vars, "l")); + scripts.put("doc['ml'].values", vars -> docScript(vars, "ml")); + scripts.put("doc['d'].values", vars -> docScript(vars, "d")); + scripts.put("doc['md'].values", vars -> docScript(vars, "md")); + scripts.put("doc['s'].values", vars -> docScript(vars, "s")); + scripts.put("doc['ms'].values", vars -> docScript(vars, "ms")); + + return scripts; + } + + @SuppressWarnings("unchecked") + static Object fieldsScript(Map<String, Object> vars, String fieldName) { + Map<?, ?> fields = (Map) vars.get("_fields"); + FieldLookup fieldLookup = (FieldLookup) fields.get(fieldName); + return fieldLookup.getValue(); + } + + @SuppressWarnings("unchecked") + static Object sourceScript(Map<String, Object> vars, String path) { + Map<String, Object> source = (Map) vars.get("_source"); + return XContentMapValues.extractValue(path, source); + } + + @SuppressWarnings("unchecked") + static Object docScript(Map<String, Object> vars, String fieldName) { + Map<?, ?> doc = (Map) vars.get("doc"); + ScriptDocValues<?> values = (ScriptDocValues<?>) doc.get(fieldName); + return values.getValues(); + } } public void testStoredFields() throws Exception { @@ -127,7 +200,12 @@ public class SearchFieldsTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3")); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*3").addStoredField("field1").addStoredField("field2").execute().actionGet(); + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addStoredField("*3") + .addStoredField("field1") + .addStoredField("field2") + .get(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); assertThat(searchResponse.getHits().hits().length, equalTo(1)); assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(2)); @@ -156,7 +234,11 @@ public class SearchFieldsTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1")); assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3")); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*").addStoredField("_source").execute().actionGet(); + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addStoredField("*") + .addStoredField("_source") + .get(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); assertThat(searchResponse.getHits().hits().length, equalTo(1)); assertThat(searchResponse.getHits().getAt(0).source(), notNullValue()); @@ -175,25 +257,37 @@ public class SearchFieldsTests extends ESIntegTestCase { client().admin().indices().preparePutMapping().setType("type1").setSource(mapping).execute().actionGet(); client().prepareIndex("test", "type1", "1") - .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).field("date", "1970-01-01T00:00:00").endObject()) + .setSource(jsonBuilder().startObject() + .field("test", "value beck") + .field("num1", 1.0f) + .field("date", "1970-01-01T00:00:00") + .endObject()) .execute().actionGet(); client().admin().indices().prepareFlush().execute().actionGet(); client().prepareIndex("test", "type1", "2") - .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).field("date", "1970-01-01T00:00:25").endObject()) - .execute().actionGet(); + .setSource(jsonBuilder().startObject() + .field("test", "value beck") + .field("num1", 2.0f) + .field("date", "1970-01-01T00:00:25") + .endObject()) + .get(); client().admin().indices().prepareFlush().execute().actionGet(); client().prepareIndex("test", "type1", "3") - .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).field("date", "1970-01-01T00:02:00").endObject()) - .execute().actionGet(); + .setSource(jsonBuilder().startObject() + .field("test", "value beck") + .field("num1", 3.0f) + .field("date", "1970-01-01T00:02:00") + .endObject()) + .get(); client().admin().indices().refresh(refreshRequest()).actionGet(); logger.info("running doc['num1'].value"); SearchResponse response = client().prepareSearch() .setQuery(matchAllQuery()) .addSort("num1", SortOrder.ASC) - .addScriptField("sNum1", new Script("doc['num1'].value")) - .addScriptField("sNum1_field", new Script("_fields['num1'].value")) - .addScriptField("date1", new Script("doc['date'].date.millis")) + .addScriptField("sNum1", new Script("doc['num1'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .addScriptField("sNum1_field", new Script("_fields['num1'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .addScriptField("date1", new Script("doc['date'].date.millis", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) .execute().actionGet(); assertNoFailures(response); @@ -204,48 +298,48 @@ public class SearchFieldsTests extends ESIntegTestCase { Set<String> fields = new HashSet<>(response.getHits().getAt(0).fields().keySet()); fields.remove(TimestampFieldMapper.NAME); // randomly enabled via templates assertThat(fields, equalTo(newHashSet("sNum1", "sNum1_field", "date1"))); - assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0)); - assertThat((Double) response.getHits().getAt(0).fields().get("sNum1_field").values().get(0), equalTo(1.0)); - assertThat((Long) response.getHits().getAt(0).fields().get("date1").values().get(0), equalTo(0L)); + assertThat(response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0)); + assertThat(response.getHits().getAt(0).fields().get("sNum1_field").values().get(0), equalTo(1.0)); + assertThat(response.getHits().getAt(0).fields().get("date1").values().get(0), equalTo(0L)); assertThat(response.getHits().getAt(1).id(), equalTo("2")); fields = new HashSet<>(response.getHits().getAt(0).fields().keySet()); fields.remove(TimestampFieldMapper.NAME); // randomly enabled via templates assertThat(fields, equalTo(newHashSet("sNum1", "sNum1_field", "date1"))); - assertThat((Double) response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0)); - assertThat((Double) response.getHits().getAt(1).fields().get("sNum1_field").values().get(0), equalTo(2.0)); - assertThat((Long) response.getHits().getAt(1).fields().get("date1").values().get(0), equalTo(25000L)); + assertThat(response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0)); + assertThat(response.getHits().getAt(1).fields().get("sNum1_field").values().get(0), equalTo(2.0)); + assertThat(response.getHits().getAt(1).fields().get("date1").values().get(0), equalTo(25000L)); assertThat(response.getHits().getAt(2).id(), equalTo("3")); fields = new HashSet<>(response.getHits().getAt(0).fields().keySet()); fields.remove(TimestampFieldMapper.NAME); // randomly enabled via templates assertThat(fields, equalTo(newHashSet("sNum1", "sNum1_field", "date1"))); - assertThat((Double) response.getHits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0)); - assertThat((Double) response.getHits().getAt(2).fields().get("sNum1_field").values().get(0), equalTo(3.0)); - assertThat((Long) response.getHits().getAt(2).fields().get("date1").values().get(0), equalTo(120000L)); + assertThat(response.getHits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0)); + assertThat(response.getHits().getAt(2).fields().get("sNum1_field").values().get(0), equalTo(3.0)); + assertThat(response.getHits().getAt(2).fields().get("date1").values().get(0), equalTo(120000L)); logger.info("running doc['num1'].value * factor"); Map<String, Object> params = MapBuilder.<String, Object>newMapBuilder().put("factor", 2.0).map(); response = client().prepareSearch() .setQuery(matchAllQuery()) .addSort("num1", SortOrder.ASC) - .addScriptField("sNum1", new Script("doc['num1'].value * factor", ScriptType.INLINE, null, params)) - .execute().actionGet(); + .addScriptField("sNum1", new Script("doc['num1'].value * factor", ScriptType.INLINE, CustomScriptPlugin.NAME, params)) + .get(); assertThat(response.getHits().totalHits(), equalTo(3L)); assertThat(response.getHits().getAt(0).id(), equalTo("1")); fields = new HashSet<>(response.getHits().getAt(0).fields().keySet()); fields.remove(TimestampFieldMapper.NAME); // randomly enabled via templates assertThat(fields, equalTo(singleton("sNum1"))); - assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0)); + assertThat(response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0)); assertThat(response.getHits().getAt(1).id(), equalTo("2")); fields = new HashSet<>(response.getHits().getAt(0).fields().keySet()); fields.remove(TimestampFieldMapper.NAME); // randomly enabled via templates assertThat(fields, equalTo(singleton("sNum1"))); - assertThat((Double) response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(4.0)); + assertThat(response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(4.0)); assertThat(response.getHits().getAt(2).id(), equalTo("3")); fields = new HashSet<>(response.getHits().getAt(0).fields().keySet()); fields.remove(TimestampFieldMapper.NAME); // randomly enabled via templates assertThat(fields, equalTo(singleton("sNum1"))); - assertThat((Double) response.getHits().getAt(2).fields().get("sNum1").values().get(0), equalTo(6.0)); + assertThat(response.getHits().getAt(2).fields().get("sNum1").values().get(0), equalTo(6.0)); } public void testUidBasedScriptFields() throws Exception { @@ -260,8 +354,11 @@ public class SearchFieldsTests extends ESIntegTestCase { indexRandom(true, indexRequestBuilders); SearchResponse response = client().prepareSearch() - .setQuery(matchAllQuery()).addSort("num1", SortOrder.ASC).setSize(numDocs) - .addScriptField("uid", new Script("_fields._uid.value")).get(); + .setQuery(matchAllQuery()) + .addSort("num1", SortOrder.ASC) + .setSize(numDocs) + .addScriptField("uid", new Script("_fields._uid.value", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .get(); assertNoFailures(response); @@ -271,12 +368,15 @@ public class SearchFieldsTests extends ESIntegTestCase { Set<String> fields = new HashSet<>(response.getHits().getAt(i).fields().keySet()); fields.remove(TimestampFieldMapper.NAME); // randomly enabled via templates assertThat(fields, equalTo(singleton("uid"))); - assertThat((String)response.getHits().getAt(i).fields().get("uid").value(), equalTo("type1#" + Integer.toString(i))); + assertThat(response.getHits().getAt(i).fields().get("uid").value(), equalTo("type1#" + Integer.toString(i))); } response = client().prepareSearch() - .setQuery(matchAllQuery()).addSort("num1", SortOrder.ASC).setSize(numDocs) - .addScriptField("id", new Script("_fields._id.value")).get(); + .setQuery(matchAllQuery()) + .addSort("num1", SortOrder.ASC) + .setSize(numDocs) + .addScriptField("id", new Script("_fields._id.value", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .get(); assertNoFailures(response); @@ -286,12 +386,15 @@ public class SearchFieldsTests extends ESIntegTestCase { Set<String> fields = new HashSet<>(response.getHits().getAt(i).fields().keySet()); fields.remove(TimestampFieldMapper.NAME); // randomly enabled via templates assertThat(fields, equalTo(singleton("id"))); - assertThat((String)response.getHits().getAt(i).fields().get("id").value(), equalTo(Integer.toString(i))); + assertThat(response.getHits().getAt(i).fields().get("id").value(), equalTo(Integer.toString(i))); } response = client().prepareSearch() - .setQuery(matchAllQuery()).addSort("num1", SortOrder.ASC).setSize(numDocs) - .addScriptField("type", new Script("_fields._type.value")).get(); + .setQuery(matchAllQuery()) + .addSort("num1", SortOrder.ASC) + .setSize(numDocs) + .addScriptField("type", new Script("_fields._type.value", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .get(); assertNoFailures(response); @@ -301,13 +404,17 @@ public class SearchFieldsTests extends ESIntegTestCase { Set<String> fields = new HashSet<>(response.getHits().getAt(i).fields().keySet()); fields.remove(TimestampFieldMapper.NAME); // randomly enabled via templates assertThat(fields, equalTo(singleton("type"))); - assertThat((String)response.getHits().getAt(i).fields().get("type").value(), equalTo("type1")); + assertThat(response.getHits().getAt(i).fields().get("type").value(), equalTo("type1")); } response = client().prepareSearch() - .setQuery(matchAllQuery()).addSort("num1", SortOrder.ASC).setSize(numDocs) - .addScriptField("id", new Script("_fields._id.value")).addScriptField("uid", new Script("_fields._uid.value")) - .addScriptField("type", new Script("_fields._type.value")).get(); + .setQuery(matchAllQuery()) + .addSort("num1", SortOrder.ASC) + .setSize(numDocs) + .addScriptField("id", new Script("_fields._id.value", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .addScriptField("uid", new Script("_fields._uid.value", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .addScriptField("type", new Script("_fields._type.value", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .get(); assertNoFailures(response); @@ -317,9 +424,9 @@ public class SearchFieldsTests extends ESIntegTestCase { Set<String> fields = new HashSet<>(response.getHits().getAt(i).fields().keySet()); fields.remove(TimestampFieldMapper.NAME); // randomly enabled via templates assertThat(fields, equalTo(newHashSet("uid", "type", "id"))); - assertThat((String)response.getHits().getAt(i).fields().get("uid").value(), equalTo("type1#" + Integer.toString(i))); - assertThat((String)response.getHits().getAt(i).fields().get("type").value(), equalTo("type1")); - assertThat((String)response.getHits().getAt(i).fields().get("id").value(), equalTo(Integer.toString(i))); + assertThat(response.getHits().getAt(i).fields().get("uid").value(), equalTo("type1#" + Integer.toString(i))); + assertThat(response.getHits().getAt(i).fields().get("type").value(), equalTo("type1")); + assertThat(response.getHits().getAt(i).fields().get("id").value(), equalTo(Integer.toString(i))); } } @@ -335,10 +442,14 @@ public class SearchFieldsTests extends ESIntegTestCase { .execute().actionGet(); client().admin().indices().refresh(refreshRequest()).actionGet(); - SearchResponse response = client().prepareSearch().setQuery(matchAllQuery()).addScriptField("s_obj1", new Script("_source.obj1")) - .addScriptField("s_obj1_test", new Script("_source.obj1.test")).addScriptField("s_obj2", new Script("_source.obj2")) - .addScriptField("s_obj2_arr2", new Script("_source.obj2.arr2")).addScriptField("s_arr3", new Script("_source.arr3")) - .execute().actionGet(); + SearchResponse response = client().prepareSearch() + .setQuery(matchAllQuery()) + .addScriptField("s_obj1", new Script("_source.obj1", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .addScriptField("s_obj1_test", new Script("_source.obj1.test", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .addScriptField("s_obj2", new Script("_source.obj2", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .addScriptField("s_obj2_arr2", new Script("_source.obj2.arr2", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .addScriptField("s_arr3", new Script("_source.arr3", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .get(); assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); @@ -365,12 +476,13 @@ public class SearchFieldsTests extends ESIntegTestCase { public void testScriptFieldsForNullReturn() throws Exception { client().prepareIndex("test", "type1", "1") - .setSource("foo", "bar") - .setRefreshPolicy("true").get(); + .setSource("foo", "bar") + .setRefreshPolicy("true").get(); - SearchResponse response = client().prepareSearch().setQuery(matchAllQuery()) - .addScriptField("test_script_1", new Script("return null")) - .get(); + SearchResponse response = client().prepareSearch() + .setQuery(matchAllQuery()) + .addScriptField("test_script_1", new Script("return null", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .get(); assertNoFailures(response); @@ -401,17 +513,53 @@ public class SearchFieldsTests extends ESIntegTestCase { public void testStoredFieldsWithoutSource() throws Exception { createIndex("test"); - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("_source").field("enabled", false).endObject().startObject("properties") - .startObject("byte_field").field("type", "byte").field("store", true).endObject() - .startObject("short_field").field("type", "short").field("store", true).endObject() - .startObject("integer_field").field("type", "integer").field("store", true).endObject() - .startObject("long_field").field("type", "long").field("store", true).endObject() - .startObject("float_field").field("type", "float").field("store", true).endObject() - .startObject("double_field").field("type", "double").field("store", true).endObject() - .startObject("date_field").field("type", "date").field("store", true).endObject() - .startObject("boolean_field").field("type", "boolean").field("store", true).endObject() - .startObject("binary_field").field("type", "binary").field("store", true).endObject() - .endObject().endObject().endObject().string(); + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("type1") + .startObject("_source") + .field("enabled", false) + .endObject() + .startObject("properties") + .startObject("byte_field") + .field("type", "byte") + .field("store", true) + .endObject() + .startObject("short_field") + .field("type", "short") + .field("store", true) + .endObject() + .startObject("integer_field") + .field("type", "integer") + .field("store", true) + .endObject() + .startObject("long_field") + .field("type", "long") + .field("store", true) + .endObject() + .startObject("float_field") + .field("type", "float") + .field("store", true) + .endObject() + .startObject("double_field") + .field("type", "double") + .field("store", true) + .endObject() + .startObject("date_field") + .field("type", "date") + .field("store", true) + .endObject() + .startObject("boolean_field") + .field("type", "boolean") + .field("store", true) + .endObject() + .startObject("binary_field") + .field("type", "binary") + .field("store", true) + .endObject() + .endObject() + .endObject() + .endObject() + .string(); client().admin().indices().preparePutMapping().setType("type1").setSource(mapping).execute().actionGet(); @@ -449,17 +597,17 @@ public class SearchFieldsTests extends ESIntegTestCase { "float_field", "double_field", "date_field", "boolean_field", "binary_field"))); - assertThat(searchResponse.getHits().getAt(0).fields().get("byte_field").value().toString(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(0).fields().get("short_field").value().toString(), equalTo("2")); - assertThat(searchResponse.getHits().getAt(0).fields().get("integer_field").value(), equalTo((Object) 3)); - assertThat(searchResponse.getHits().getAt(0).fields().get("long_field").value(), equalTo((Object) 4L)); - assertThat(searchResponse.getHits().getAt(0).fields().get("float_field").value(), equalTo((Object) 5.0f)); - assertThat(searchResponse.getHits().getAt(0).fields().get("double_field").value(), equalTo((Object) 6.0d)); + SearchHit searchHit = searchResponse.getHits().getAt(0); + assertThat(searchHit.fields().get("byte_field").value().toString(), equalTo("1")); + assertThat(searchHit.fields().get("short_field").value().toString(), equalTo("2")); + assertThat(searchHit.fields().get("integer_field").value(), equalTo((Object) 3)); + assertThat(searchHit.fields().get("long_field").value(), equalTo((Object) 4L)); + assertThat(searchHit.fields().get("float_field").value(), equalTo((Object) 5.0f)); + assertThat(searchHit.fields().get("double_field").value(), equalTo((Object) 6.0d)); String dateTime = Joda.forPattern("dateOptionalTime").printer().print(new DateTime(2012, 3, 22, 0, 0, DateTimeZone.UTC)); - assertThat(searchResponse.getHits().getAt(0).fields().get("date_field").value(), equalTo((Object) dateTime)); - assertThat(searchResponse.getHits().getAt(0).fields().get("boolean_field").value(), equalTo((Object) Boolean.TRUE)); - assertThat(((BytesReference) searchResponse.getHits().getAt(0).fields().get("binary_field").value()), equalTo((BytesReference) new BytesArray("testing text".getBytes("UTF8")))); - + assertThat(searchHit.fields().get("date_field").value(), equalTo((Object) dateTime)); + assertThat(searchHit.fields().get("boolean_field").value(), equalTo((Object) Boolean.TRUE)); + assertThat(searchHit.fields().get("binary_field").value(), equalTo(new BytesArray("testing text" .getBytes("UTF8")))); } public void testSearchFieldsMetaData() throws Exception { @@ -575,25 +723,57 @@ public class SearchFieldsTests extends ESIntegTestCase { new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).fieldDataField("test_field")).get(); assertHitCount(searchResponse, 1); Map<String,SearchHitField> fields = searchResponse.getHits().getHits()[0].getFields(); - assertThat((String)fields.get("test_field").value(), equalTo("foobar")); + assertThat(fields.get("test_field").value(), equalTo("foobar")); } public void testFieldsPulledFromFieldData() throws Exception { createIndex("test"); - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("_source").field("enabled", false).endObject().startObject("properties") - .startObject("text_field").field("type", "text").field("fielddata", true).endObject() - .startObject("keyword_field").field("type", "keyword").endObject() - .startObject("byte_field").field("type", "byte").endObject() - .startObject("short_field").field("type", "short").endObject() - .startObject("integer_field").field("type", "integer").endObject() - .startObject("long_field").field("type", "long").endObject() - .startObject("float_field").field("type", "float").endObject() - .startObject("double_field").field("type", "double").endObject() - .startObject("date_field").field("type", "date").endObject() - .startObject("boolean_field").field("type", "boolean").endObject() - .startObject("binary_field").field("type", "binary").endObject() - .endObject().endObject().endObject().string(); + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("type1") + .startObject("_source") + .field("enabled", false) + .endObject() + .startObject("properties") + .startObject("text_field") + .field("type", "text") + .field("fielddata", true) + .endObject() + .startObject("keyword_field") + .field("type", "keyword") + .endObject() + .startObject("byte_field") + .field("type", "byte") + .endObject() + .startObject("short_field") + .field("type", "short") + .endObject() + .startObject("integer_field") + .field("type", "integer") + .endObject() + .startObject("long_field") + .field("type", "long") + .endObject() + .startObject("float_field") + .field("type", "float") + .endObject() + .startObject("double_field") + .field("type", "double") + .endObject() + .startObject("date_field") + .field("type", "date") + .endObject() + .startObject("boolean_field") + .field("type", "boolean") + .endObject() + .startObject("binary_field") + .field("type", "binary") + .endObject() + .endObject() + .endObject() + .endObject() + .string(); client().admin().indices().preparePutMapping().setType("type1").setSource(mapping).execute().actionGet(); @@ -667,7 +847,7 @@ public class SearchFieldsTests extends ESIntegTestCase { ensureSearchable(); SearchRequestBuilder req = client().prepareSearch("index"); for (String field : Arrays.asList("s", "ms", "l", "ml", "d", "md")) { - req.addScriptField(field, new Script("doc['" + field + "'].values")); + req.addScriptField(field, new Script("doc['" + field + "'].values", ScriptType.INLINE, CustomScriptPlugin.NAME, null)); } SearchResponse resp = req.get(); assertSearchResponse(resp); @@ -690,11 +870,11 @@ public class SearchFieldsTests extends ESIntegTestCase { indexRandom(true, client().prepareIndex("test", "my-type1", "1") - .setRouting("1") - .setTimestamp("205097") - .setTTL(10000000000000L) - .setParent("parent_1") - .setSource(jsonBuilder().startObject().field("field1", "value").endObject())); + .setRouting("1") + .setTimestamp("205097") + .setTTL(10000000000000L) + .setParent("parent_1") + .setSource(jsonBuilder().startObject().field("field1", "value").endObject())); SearchResponse response = client().prepareSearch("test").addStoredField("field1").get(); assertSearchResponse(response); diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java b/core/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java similarity index 66% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java rename to core/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java index 6afa738569c..a5eb37f67b1 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java @@ -16,26 +16,27 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.functionscore; import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.MockScriptPlugin; +import org.elasticsearch.script.ScoreAccessor; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.CoreMatchers; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.Map; +import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; @@ -44,6 +45,7 @@ import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction; import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.randomFunction; import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; +import static org.elasticsearch.script.MockScriptPlugin.NAME; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.allOf; @@ -54,11 +56,41 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.nullValue; -public class RandomScoreFunctionTests extends ESIntegTestCase { +public class RandomScoreFunctionIT extends ESIntegTestCase { @Override protected Collection<Class<? extends Plugin>> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); + return pluginList(CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends MockScriptPlugin { + + @Override + @SuppressWarnings("unchecked") + protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() { + Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>(); + + scripts.put("log(doc['index'].value + (factor * _score))", + vars -> scoringScript(vars, ScoreAccessor::doubleValue)); + scripts.put("log(doc['index'].value + (factor * _score.intValue()))", + vars -> scoringScript(vars, ScoreAccessor::intValue)); + scripts.put("log(doc['index'].value + (factor * _score.longValue()))", + vars -> scoringScript(vars, ScoreAccessor::longValue)); + scripts.put("log(doc['index'].value + (factor * _score.floatValue()))", + vars -> scoringScript(vars, ScoreAccessor::floatValue)); + scripts.put("log(doc['index'].value + (factor * _score.doubleValue()))", + vars -> scoringScript(vars, ScoreAccessor::doubleValue)); + return scripts; + } + + @SuppressWarnings("unchecked") + static Double scoringScript(Map<String, Object> vars, Function<ScoreAccessor, Number> scoring) { + Map<?, ?> doc = (Map) vars.get("doc"); + Double index = ((Number) ((ScriptDocValues<?>) doc.get("index")).getValues().get(0)).doubleValue(); + Double score = scoring.apply((ScoreAccessor) vars.get("_score")).doubleValue(); + Integer factor = (Integer) vars.get("factor"); + return Math.log(index + (factor * score)); + } } public void testConsistentHitsWithSameSeed() throws Exception { @@ -86,17 +118,15 @@ public class RandomScoreFunctionTests extends ESIntegTestCase { .setPreference(preference) .setQuery(functionScoreQuery(matchAllQuery(), randomFunction(seed))) .execute().actionGet(); - assertThat("Failures " + Arrays.toString(searchResponse.getShardFailures()), searchResponse.getShardFailures().length, CoreMatchers.equalTo(0)); + assertThat("Failures " + Arrays.toString(searchResponse.getShardFailures()), + searchResponse.getShardFailures().length, CoreMatchers.equalTo(0)); final int hitCount = searchResponse.getHits().getHits().length; final SearchHit[] currentHits = searchResponse.getHits().getHits(); - ArrayUtil.timSort(currentHits, new Comparator<SearchHit>() { - @Override - public int compare(SearchHit o1, SearchHit o2) { - // for tie-breaking we have to resort here since if the score is - // identical we rely on collection order which might change. - int cmp = Float.compare(o1.getScore(), o2.getScore()); - return cmp == 0 ? o1.getId().compareTo(o2.getId()) : cmp; - } + ArrayUtil.timSort(currentHits, (o1, o2) -> { + // for tie-breaking we have to resort here since if the score is + // identical we rely on collection order which might change. + int cmp = Float.compare(o1.getScore(), o2.getScore()); + return cmp == 0 ? o1.getId().compareTo(o2.getId()) : cmp; }); if (i == 0) { assertThat(hits, nullValue()); @@ -128,71 +158,92 @@ public class RandomScoreFunctionTests extends ESIntegTestCase { int docCount = randomIntBetween(100, 200); for (int i = 0; i < docCount; i++) { - client().prepareIndex("test", "type", "" + i).setSource("body", randomFrom(Arrays.asList("foo", "bar", "baz")), "index", i + 1) // we add 1 to the index field to make sure that the scripts below never compute log(0) + client().prepareIndex("test", "type", "" + i) + // we add 1 to the index field to make sure that the scripts below never compute log(0) + .setSource("body", randomFrom(Arrays.asList("foo", "bar", "baz")), "index", i + 1) .get(); } refresh(); Map<String, Object> params = new HashMap<>(); params.put("factor", randomIntBetween(2, 4)); + // Test for accessing _score + Script script = new Script("log(doc['index'].value + (factor * _score))", ScriptType.INLINE, NAME, params); SearchResponse resp = client() .prepareSearch("test") .setQuery( - functionScoreQuery(matchQuery("body", "foo"), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ - new FunctionScoreQueryBuilder.FilterFunctionBuilder(fieldValueFactorFunction("index").factor(2)), - new FunctionScoreQueryBuilder.FilterFunctionBuilder(scriptFunction(new Script("log(doc['index'].value + (factor * _score))", ScriptType.INLINE, null, params))) - })).get(); + functionScoreQuery(matchQuery("body", "foo"), + new FunctionScoreQueryBuilder.FilterFunctionBuilder[] { + new FunctionScoreQueryBuilder.FilterFunctionBuilder(fieldValueFactorFunction("index").factor(2)), + new FunctionScoreQueryBuilder.FilterFunctionBuilder(scriptFunction(script)) + } + )) + .get(); assertNoFailures(resp); SearchHit firstHit = resp.getHits().getAt(0); assertThat(firstHit.getScore(), greaterThan(1f)); // Test for accessing _score.intValue() + script = new Script("log(doc['index'].value + (factor * _score.intValue()))", ScriptType.INLINE, NAME, params); resp = client() .prepareSearch("test") .setQuery( - functionScoreQuery(matchQuery("body", "foo"), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ - new FunctionScoreQueryBuilder.FilterFunctionBuilder(fieldValueFactorFunction("index").factor(2)), - new FunctionScoreQueryBuilder.FilterFunctionBuilder(scriptFunction(new Script("log(doc['index'].value + (factor * _score.intValue()))", ScriptType.INLINE, null, params))) - })).get(); + functionScoreQuery(matchQuery("body", "foo"), + new FunctionScoreQueryBuilder.FilterFunctionBuilder[] { + new FunctionScoreQueryBuilder.FilterFunctionBuilder(fieldValueFactorFunction("index").factor(2)), + new FunctionScoreQueryBuilder.FilterFunctionBuilder(scriptFunction(script)) + } + )) + .get(); assertNoFailures(resp); firstHit = resp.getHits().getAt(0); assertThat(firstHit.getScore(), greaterThan(1f)); // Test for accessing _score.longValue() + script = new Script("log(doc['index'].value + (factor * _score.longValue()))", ScriptType.INLINE, NAME, params); resp = client() .prepareSearch("test") .setQuery( - functionScoreQuery(matchQuery("body", "foo"), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ - new FunctionScoreQueryBuilder.FilterFunctionBuilder(fieldValueFactorFunction("index").factor(2)), - new FunctionScoreQueryBuilder.FilterFunctionBuilder(scriptFunction(new Script("log(doc['index'].value + (factor * _score.longValue()))", ScriptType.INLINE, null, params))) - })).get(); + functionScoreQuery(matchQuery("body", "foo"), + new FunctionScoreQueryBuilder.FilterFunctionBuilder[] { + new FunctionScoreQueryBuilder.FilterFunctionBuilder(fieldValueFactorFunction("index").factor(2)), + new FunctionScoreQueryBuilder.FilterFunctionBuilder(scriptFunction(script)) + } + )) + .get(); assertNoFailures(resp); firstHit = resp.getHits().getAt(0); assertThat(firstHit.getScore(), greaterThan(1f)); // Test for accessing _score.floatValue() + script = new Script("log(doc['index'].value + (factor * _score.floatValue()))", ScriptType.INLINE, NAME, params); resp = client() .prepareSearch("test") .setQuery( - functionScoreQuery(matchQuery("body", "foo"), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ - new FunctionScoreQueryBuilder.FilterFunctionBuilder(fieldValueFactorFunction("index").factor(2)), - new FunctionScoreQueryBuilder.FilterFunctionBuilder(scriptFunction(new Script("log(doc['index'].value + (factor * _score.floatValue()))", - ScriptType.INLINE, null, params))) - })).get(); + functionScoreQuery(matchQuery("body", "foo"), + new FunctionScoreQueryBuilder.FilterFunctionBuilder[] { + new FunctionScoreQueryBuilder.FilterFunctionBuilder(fieldValueFactorFunction("index").factor(2)), + new FunctionScoreQueryBuilder.FilterFunctionBuilder(scriptFunction(script)) + } + )) + .get(); assertNoFailures(resp); firstHit = resp.getHits().getAt(0); assertThat(firstHit.getScore(), greaterThan(1f)); // Test for accessing _score.doubleValue() + script = new Script("log(doc['index'].value + (factor * _score.doubleValue()))", ScriptType.INLINE, NAME, params); resp = client() .prepareSearch("test") .setQuery( - functionScoreQuery(matchQuery("body", "foo"), new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{ + functionScoreQuery(matchQuery("body", "foo"), + new FunctionScoreQueryBuilder.FilterFunctionBuilder[] { new FunctionScoreQueryBuilder.FilterFunctionBuilder(fieldValueFactorFunction("index").factor(2)), - new FunctionScoreQueryBuilder.FilterFunctionBuilder(scriptFunction(new Script("log(doc['index'].value + (factor * _score.doubleValue()))", - ScriptType.INLINE, null, params))) - })).get(); + new FunctionScoreQueryBuilder.FilterFunctionBuilder(scriptFunction(script)) + } + )) + .get(); assertNoFailures(resp); firstHit = resp.getHits().getAt(0); assertThat(firstHit.getScore(), greaterThan(1f)); @@ -208,9 +259,9 @@ public class RandomScoreFunctionTests extends ESIntegTestCase { int seed = 12345678; SearchResponse resp = client().prepareSearch("test") - .setQuery(functionScoreQuery(matchAllQuery(), randomFunction(seed))) - .setExplain(true) - .get(); + .setQuery(functionScoreQuery(matchAllQuery(), randomFunction(seed))) + .setExplain(true) + .get(); assertNoFailures(resp); assertEquals(1, resp.getHits().totalHits()); SearchHit firstHit = resp.getHits().getAt(0); @@ -222,8 +273,8 @@ public class RandomScoreFunctionTests extends ESIntegTestCase { ensureGreen(); SearchResponse resp = client().prepareSearch("test") - .setQuery(functionScoreQuery(matchAllQuery(), randomFunction(1234))) - .get(); + .setQuery(functionScoreQuery(matchAllQuery(), randomFunction(1234))) + .get(); assertNoFailures(resp); assertEquals(0, resp.getHits().totalHits()); } @@ -243,9 +294,9 @@ public class RandomScoreFunctionTests extends ESIntegTestCase { for (int i = 0; i < iters; ++i) { int seed = randomInt(); SearchResponse searchResponse = client().prepareSearch() - .setQuery(functionScoreQuery(matchAllQuery(), randomFunction(seed))) - .setSize(docCount) - .execute().actionGet(); + .setQuery(functionScoreQuery(matchAllQuery(), randomFunction(seed))) + .setSize(docCount) + .execute().actionGet(); assertNoFailures(searchResponse); for (SearchHit hit : searchResponse.getHits().getHits()) { @@ -264,19 +315,19 @@ public class RandomScoreFunctionTests extends ESIntegTestCase { flushAndRefresh(); assertNoFailures(client().prepareSearch() - .setSize(docCount) // get all docs otherwise we are prone to tie-breaking - .setQuery(functionScoreQuery(matchAllQuery(), randomFunction(randomInt()))) - .execute().actionGet()); + .setSize(docCount) // get all docs otherwise we are prone to tie-breaking + .setQuery(functionScoreQuery(matchAllQuery(), randomFunction(randomInt()))) + .execute().actionGet()); assertNoFailures(client().prepareSearch() - .setSize(docCount) // get all docs otherwise we are prone to tie-breaking - .setQuery(functionScoreQuery(matchAllQuery(), randomFunction(randomLong()))) - .execute().actionGet()); + .setSize(docCount) // get all docs otherwise we are prone to tie-breaking + .setQuery(functionScoreQuery(matchAllQuery(), randomFunction(randomLong()))) + .execute().actionGet()); assertNoFailures(client().prepareSearch() - .setSize(docCount) // get all docs otherwise we are prone to tie-breaking - .setQuery(functionScoreQuery(matchAllQuery(), randomFunction(randomRealisticUnicodeOfLengthBetween(10, 20)))) - .execute().actionGet()); + .setSize(docCount) // get all docs otherwise we are prone to tie-breaking + .setQuery(functionScoreQuery(matchAllQuery(), randomFunction(randomRealisticUnicodeOfLengthBetween(10, 20)))) + .execute().actionGet()); } public void checkDistribution() throws Exception { @@ -344,5 +395,4 @@ public class RandomScoreFunctionTests extends ESIntegTestCase { logger.info("mean: {}", sum / (double) count); } - } diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java b/core/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java similarity index 52% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java rename to core/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java index 0d7dd4e12e2..cb574ce46f3 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java +++ b/core/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.geo; import org.elasticsearch.Version; import org.elasticsearch.action.search.SearchResponse; @@ -27,32 +27,71 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; -import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.VersionUtils; import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.script.ScriptService.ScriptType; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.closeTo; -/** - */ -public class GeoDistanceTests extends ESIntegTestCase { +public class GeoDistanceIT extends ESIntegTestCase { + + private static final double source_lat = 32.798; + private static final double source_long = -117.151; + private static final double target_lat = 32.81; + private static final double target_long = -117.21; + @Override protected Collection<Class<? extends Plugin>> nodePlugins() { - return pluginList(GroovyPlugin.class, InternalSettingsPlugin.class); + return pluginList(CustomScriptPlugin.class, InternalSettingsPlugin.class); + } + + public static class CustomScriptPlugin extends MockScriptPlugin { + + @Override + @SuppressWarnings("unchecked") + protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() { + Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>(); + + scripts.put("arcDistance", vars -> distanceScript(vars, + location -> location.arcDistance(target_lat, target_long))); + scripts.put("distance", vars -> distanceScript(vars, + location -> location.distance(target_lat, target_long))); + scripts.put("arcDistanceInKm", vars -> distanceScript(vars, + location -> location.arcDistanceInKm(target_lat, target_long))); + scripts.put("distanceInKm", vars -> distanceScript(vars, + location -> location.distanceInKm(target_lat, target_long))); + scripts.put("arcDistanceInKm(lat, lon + 360)", vars -> distanceScript(vars, + location -> location.arcDistanceInKm(target_lat, target_long + 360))); + scripts.put("arcDistanceInKm(lat + 360, lon)", vars -> distanceScript(vars, + location -> location.arcDistanceInKm(target_lat + 360, target_long))); + scripts.put("arcDistanceInMiles", vars -> distanceScript(vars, + location -> location.arcDistanceInMiles(target_lat, target_long))); + scripts.put("distanceInMiles", vars -> distanceScript(vars, + location -> location.distanceInMiles(target_lat, target_long))); + + return scripts; + } + + @SuppressWarnings("unchecked") + static Double distanceScript(Map<String, Object> vars, Function<ScriptDocValues.GeoPoints, Double> distance) { + Map<?, ?> doc = (Map) vars.get("doc"); + return distance.apply((ScriptDocValues.GeoPoints) doc.get("location")); + } } public void testDistanceScript() throws Exception { - double source_lat = 32.798; - double source_long = -117.151; - double target_lat = 32.81; - double target_long = -117.21; Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); @@ -65,69 +104,78 @@ public class GeoDistanceTests extends ESIntegTestCase { assertAcked(prepareCreate("test").setSettings(settings).addMapping("type1", xContentBuilder)); ensureGreen(); - client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() - .field("name", "TestPosition") - .startObject("location").field("lat", source_lat).field("lon", source_long).endObject() - .endObject()).execute().actionGet(); + client().prepareIndex("test", "type1", "1") + .setSource(jsonBuilder().startObject() + .field("name", "TestPosition") + .startObject("location") + .field("lat", source_lat) + .field("lon", source_long) + .endObject() + .endObject()) + .get(); refresh(); + // Test doc['location'].arcDistance(lat, lon) SearchResponse searchResponse1 = client().prepareSearch().addStoredField("_source") - .addScriptField("distance", new Script("doc['location'].arcDistance(" + target_lat + "," + target_long + ")")).execute() - .actionGet(); + .addScriptField("distance", new Script("arcDistance", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .get(); Double resultDistance1 = searchResponse1.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultDistance1, closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.DEFAULT), 0.01d)); + // Test doc['location'].distance(lat, lon) SearchResponse searchResponse2 = client().prepareSearch().addStoredField("_source") - .addScriptField("distance", new Script("doc['location'].distance(" + target_lat + "," + target_long + ")")).execute() - .actionGet(); + .addScriptField("distance", new Script("distance", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .get(); Double resultDistance2 = searchResponse2.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultDistance2, closeTo(GeoDistance.PLANE.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.DEFAULT), 0.01d)); + // Test doc['location'].arcDistanceInKm(lat, lon) SearchResponse searchResponse3 = client().prepareSearch().addStoredField("_source") - .addScriptField("distance", new Script("doc['location'].arcDistanceInKm(" + target_lat + "," + target_long + ")")) - .execute().actionGet(); + .addScriptField("distance", new Script("arcDistanceInKm", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .get(); Double resultArcDistance3 = searchResponse3.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultArcDistance3, closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.01d)); + // Test doc['location'].distanceInKm(lat, lon) SearchResponse searchResponse4 = client().prepareSearch().addStoredField("_source") - .addScriptField("distance", new Script("doc['location'].distanceInKm(" + target_lat + "," + target_long + ")")).execute() - .actionGet(); + .addScriptField("distance", new Script("distanceInKm", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .get(); Double resultDistance4 = searchResponse4.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultDistance4, closeTo(GeoDistance.PLANE.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.01d)); - SearchResponse searchResponse5 = client() - .prepareSearch() - .addStoredField("_source") - .addScriptField("distance", new Script("doc['location'].arcDistanceInKm(" + (target_lat) + "," + (target_long + 360) + ")")) - .execute().actionGet(); + // Test doc['location'].arcDistanceInKm(lat, lon + 360) + SearchResponse searchResponse5 = client().prepareSearch().addStoredField("_source") + .addScriptField("distance", new Script("arcDistanceInKm(lat, lon + 360)", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .get(); Double resultArcDistance5 = searchResponse5.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultArcDistance5, closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.01d)); - SearchResponse searchResponse6 = client() - .prepareSearch() - .addStoredField("_source") - .addScriptField("distance", new Script("doc['location'].arcDistanceInKm(" + (target_lat + 360) + "," + (target_long) + ")")) - .execute().actionGet(); + // Test doc['location'].arcDistanceInKm(lat + 360, lon) + SearchResponse searchResponse6 = client().prepareSearch().addStoredField("_source") + .addScriptField("distance", new Script("arcDistanceInKm(lat + 360, lon)", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .get(); Double resultArcDistance6 = searchResponse6.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultArcDistance6, closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.01d)); + // Test doc['location'].arcDistanceInMiles(lat, lon) SearchResponse searchResponse7 = client().prepareSearch().addStoredField("_source") - .addScriptField("distance", new Script("doc['location'].arcDistanceInMiles(" + target_lat + "," + target_long + ")")) - .execute().actionGet(); + .addScriptField("distance", new Script("arcDistanceInMiles", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .get(); Double resultDistance7 = searchResponse7.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultDistance7, closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.MILES), 0.01d)); + // Test doc['location'].distanceInMiles(lat, lon) SearchResponse searchResponse8 = client().prepareSearch().addStoredField("_source") - .addScriptField("distance", new Script("doc['location'].distanceInMiles(" + target_lat + "," + target_long + ")")) - .execute().actionGet(); + .addScriptField("distance", new Script("distanceInMiles", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .get(); Double resultDistance8 = searchResponse8.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultDistance8, closeTo(GeoDistance.PLANE.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.MILES), 0.01d)); diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoShapeIntegrationTests.java b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java similarity index 91% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoShapeIntegrationTests.java rename to core/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java index 73943dfaaba..4294547e7fc 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoShapeIntegrationTests.java +++ b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeIntegrationIT.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.geo; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; @@ -26,25 +26,13 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.test.ESIntegTestCase; -import java.util.Collection; -import java.util.Collections; - import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -/** - */ -public class GeoShapeIntegrationTests extends ESIntegTestCase { - - @Override - protected Collection<Class<? extends Plugin>> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); - } +public class GeoShapeIntegrationIT extends ESIntegTestCase { /** * Test that orientation parameter correctly persists across cluster restart diff --git a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java index 780e8cbcad4..974d0ade282 100644 --- a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java +++ b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java @@ -83,7 +83,7 @@ public class InnerHitsIT extends ESIntegTestCase { public static class CustomScriptPlugin extends MockScriptPlugin { @Override protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() { - return Collections.emptyMap(); + return Collections.singletonMap("5", script -> "5"); } } diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java b/core/src/test/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java similarity index 68% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java rename to core/src/test/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java index e1620c8f619..6422bf7a134 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java +++ b/core/src/test/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java @@ -17,15 +17,16 @@ * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.scriptfilter; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; @@ -34,19 +35,47 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.scriptQuery; import static org.hamcrest.Matchers.equalTo; -/** - * - */ @ESIntegTestCase.ClusterScope(scope= ESIntegTestCase.Scope.SUITE) -public class ScriptQuerySearchTests extends ESIntegTestCase { +public class ScriptQuerySearchIT extends ESIntegTestCase { + @Override protected Collection<Class<? extends Plugin>> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); + return Collections.singleton(CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends MockScriptPlugin { + + @Override + protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() { + Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>(); + + scripts.put("doc['num1'].value", vars -> { + Map<?, ?> doc = (Map) vars.get("doc"); + return doc.get("num1"); + }); + + scripts.put("doc['num1'].value > 1", vars -> { + Map<?, ?> doc = (Map) vars.get("doc"); + ScriptDocValues.Doubles num1 = (ScriptDocValues.Doubles) doc.get("num1"); + return num1.getValue() > 1; + }); + + scripts.put("doc['num1'].value > param1", vars -> { + Integer param1 = (Integer) vars.get("param1"); + + Map<?, ?> doc = (Map) vars.get("doc"); + ScriptDocValues.Doubles num1 = (ScriptDocValues.Doubles) doc.get("num1"); + return num1.getValue() > param1; + }); + + return scripts; + } } @Override @@ -62,21 +91,23 @@ public class ScriptQuerySearchTests extends ESIntegTestCase { createIndex("test"); client().prepareIndex("test", "type1", "1") .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject()) - .execute().actionGet(); + .get(); flush(); client().prepareIndex("test", "type1", "2") .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject()) - .execute().actionGet(); + .get(); flush(); client().prepareIndex("test", "type1", "3") .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).endObject()) - .execute().actionGet(); + .get(); refresh(); logger.info("running doc['num1'].value > 1"); SearchResponse response = client().prepareSearch() - .setQuery(scriptQuery(new Script("doc['num1'].value > 1"))).addSort("num1", SortOrder.ASC) - .addScriptField("sNum1", new Script("doc['num1'].value")).execute().actionGet(); + .setQuery(scriptQuery(new Script("doc['num1'].value > 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null))) + .addSort("num1", SortOrder.ASC) + .addScriptField("sNum1", new Script("doc['num1'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .get(); assertThat(response.getHits().totalHits(), equalTo(2L)); assertThat(response.getHits().getAt(0).id(), equalTo("2")); @@ -90,8 +121,10 @@ public class ScriptQuerySearchTests extends ESIntegTestCase { logger.info("running doc['num1'].value > param1"); response = client() .prepareSearch() - .setQuery(scriptQuery(new Script("doc['num1'].value > param1", ScriptType.INLINE, null, params))) - .addSort("num1", SortOrder.ASC).addScriptField("sNum1", new Script("doc['num1'].value")).execute().actionGet(); + .setQuery(scriptQuery(new Script("doc['num1'].value > param1", ScriptType.INLINE, CustomScriptPlugin.NAME, params))) + .addSort("num1", SortOrder.ASC) + .addScriptField("sNum1", new Script("doc['num1'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .get(); assertThat(response.getHits().totalHits(), equalTo(1L)); assertThat(response.getHits().getAt(0).id(), equalTo("3")); @@ -102,9 +135,10 @@ public class ScriptQuerySearchTests extends ESIntegTestCase { logger.info("running doc['num1'].value > param1"); response = client() .prepareSearch() - .setQuery( - scriptQuery(new Script("doc['num1'].value > param1", ScriptType.INLINE, null, params))) - .addSort("num1", SortOrder.ASC).addScriptField("sNum1", new Script("doc['num1'].value")).execute().actionGet(); + .setQuery(scriptQuery(new Script("doc['num1'].value > param1", ScriptType.INLINE, CustomScriptPlugin.NAME, params))) + .addSort("num1", SortOrder.ASC) + .addScriptField("sNum1", new Script("doc['num1'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .get(); assertThat(response.getHits().totalHits(), equalTo(3L)); assertThat(response.getHits().getAt(0).id(), equalTo("1")); diff --git a/core/src/test/java/org/elasticsearch/search/sort/SimpleSortIT.java b/core/src/test/java/org/elasticsearch/search/sort/SimpleSortIT.java new file mode 100644 index 00000000000..53cf4d07e99 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/sort/SimpleSortIT.java @@ -0,0 +1,476 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.sort; + + +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.MockScriptPlugin; +import org.elasticsearch.script.Script; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.function.Function; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.script.ScriptService.ScriptType; +import static org.elasticsearch.search.sort.SortBuilders.scriptSort; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + +public class SimpleSortIT extends ESIntegTestCase { + + private static final String DOUBLE_APOSTROPHE = "\u0027\u0027"; + + @Override + protected Collection<Class<? extends Plugin>> nodePlugins() { + return pluginList(CustomScriptPlugin.class, InternalSettingsPlugin.class); + } + + public static class CustomScriptPlugin extends MockScriptPlugin { + + @Override + protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() { + Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>(); + + scripts.put("doc['str_value'].value", vars -> { + Map<?, ?> doc = (Map) vars.get("doc"); + return ((ScriptDocValues.Strings) doc.get("str_value")).getValue(); + }); + + scripts.put("doc['id'].value", vars -> { + Map<?, ?> doc = (Map) vars.get("doc"); + return ((ScriptDocValues.Strings) doc.get("id")).getValue(); + }); + + scripts.put("doc['id'].values[0]", vars -> { + Map<?, ?> doc = (Map) vars.get("doc"); + return ((ScriptDocValues.Strings) doc.get("id")).getValues().get(0); + }); + + scripts.put("get min long", vars -> getMinValueScript(vars, Long.MAX_VALUE, "lvalue", l -> (Long) l)); + scripts.put("get min double", vars -> getMinValueScript(vars, Double.MAX_VALUE, "dvalue", d -> (Double) d)); + scripts.put("get min string", vars -> getMinValueScript(vars, Integer.MAX_VALUE, "svalue", s -> Integer.parseInt((String) s))); + scripts.put("get min geopoint lon", vars -> getMinValueScript(vars, Double.MAX_VALUE, "gvalue", g -> ((GeoPoint) g).getLon())); + + scripts.put(DOUBLE_APOSTROPHE, vars -> DOUBLE_APOSTROPHE); + + return scripts; + } + + /** + * Return the minimal value from a set of values. + */ + @SuppressWarnings("unchecked") + static <T extends Comparable<T>> T getMinValueScript(Map<String, Object> vars, T initialValue, String fieldName, + Function<Object, T> converter) { + T retval = initialValue; + Map<?, ?> doc = (Map) vars.get("doc"); + ScriptDocValues<?> values = (ScriptDocValues<?>) doc.get(fieldName); + for (Object v : values.getValues()) { + T value = converter.apply(v); + retval = (value.compareTo(retval) < 0) ? value : retval; + } + return retval; + } + } + + public void testSimpleSorts() throws Exception { + Random random = random(); + assertAcked(prepareCreate("test") + .addMapping("type1", jsonBuilder() + .startObject() + .startObject("type1") + .startObject("properties") + .startObject("str_value") + .field("type", "keyword") + .endObject() + .startObject("boolean_value") + .field("type", "boolean") + .endObject() + .startObject("byte_value") + .field("type", "byte") + .endObject() + .startObject("short_value") + .field("type", "short") + .endObject() + .startObject("integer_value") + .field("type", "integer") + .endObject() + .startObject("long_value") + .field("type", "long") + .endObject() + .startObject("float_value") + .field("type", "float") + .endObject() + .startObject("double_value") + .field("type", "double") + .endObject() + .endObject() + .endObject() + .endObject())); + ensureGreen(); + List<IndexRequestBuilder> builders = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + builders.add(client().prepareIndex("test", "type1", Integer.toString(i)) + .setSource(jsonBuilder() + .startObject() + .field("str_value", new String(new char[]{(char) (97 + i), (char) (97 + i)})) + .field("boolean_value", true) + .field("byte_value", i) + .field("short_value", i) + .field("integer_value", i) + .field("long_value", i) + .field("float_value", 0.1 * i) + .field("double_value", 0.1 * i) + .endObject() + )); + } + Collections.shuffle(builders, random); + for (IndexRequestBuilder builder : builders) { + builder.execute().actionGet(); + if (random.nextBoolean()) { + if (random.nextInt(5) != 0) { + refresh(); + } else { + client().admin().indices().prepareFlush().get(); + } + } + } + refresh(); + + // STRING script + int size = 1 + random.nextInt(10); + + Script script = new Script("doc['str_value'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + + SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(size) + .addSort(new ScriptSortBuilder(script, ScriptSortType.STRING)) + .get(); + + assertHitCount(searchResponse, 10); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + for (int i = 0; i < size; i++) { + SearchHit searchHit = searchResponse.getHits().getAt(i); + assertThat(searchHit.id(), equalTo(Integer.toString(i))); + + String expected = new String(new char[]{(char) (97 + i), (char) (97 + i)}); + assertThat(searchHit.sortValues()[0].toString(), equalTo(expected)); + } + + size = 1 + random.nextInt(10); + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(size) + .addSort("str_value", SortOrder.DESC) + .get(); + + assertHitCount(searchResponse, 10); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + for (int i = 0; i < size; i++) { + SearchHit searchHit = searchResponse.getHits().getAt(i); + assertThat(searchHit.id(), equalTo(Integer.toString(9 - i))); + + String expected = new String(new char[]{(char) (97 + (9 - i)), (char) (97 + (9 - i))}); + assertThat(searchHit.sortValues()[0].toString(), equalTo(expected)); + } + + assertThat(searchResponse.toString(), not(containsString("error"))); + assertNoFailures(searchResponse); + } + + public void testSortMinValueScript() throws IOException { + String mapping = jsonBuilder() + .startObject() + .startObject("type1") + .startObject("properties") + .startObject("lvalue") + .field("type", "long") + .endObject() + .startObject("dvalue") + .field("type", "double") + .endObject() + .startObject("svalue") + .field("type", "keyword") + .endObject() + .startObject("gvalue") + .field("type", "geo_point") + .endObject() + .endObject() + .endObject() + .endObject().string(); + + assertAcked(prepareCreate("test").addMapping("type1", mapping)); + ensureGreen(); + + for (int i = 0; i < 10; i++) { + client().prepareIndex("test", "type1", "" + i) + .setSource(jsonBuilder() + .startObject() + .field("ord", i) + .field("svalue", new String[]{"" + i, "" + (i + 1), "" + (i + 2)}) + .field("lvalue", new long[]{i, i + 1, i + 2}) + .field("dvalue", new double[]{i, i + 1, i + 2}) + .startObject("gvalue") + .field("lat", (double) i + 1) + .field("lon", (double) i) + .endObject() + .endObject()) + .get(); + } + + for (int i = 10; i < 20; i++) { // add some docs that don't have values in those fields + client().prepareIndex("test", "type1", "" + i) + .setSource(jsonBuilder() + .startObject() + .field("ord", i) + .endObject()) + .get(); + } + client().admin().indices().prepareRefresh("test").get(); + + // test the long values + SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addScriptField("min", new Script("get min long", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .addSort(SortBuilders.fieldSort("ord").order(SortOrder.ASC).unmappedType("long")) + .setSize(10) + .get(); + + assertNoFailures(searchResponse); + + assertHitCount(searchResponse, 20L); + for (int i = 0; i < 10; i++) { + SearchHit searchHit = searchResponse.getHits().getAt(i); + assertThat("res: " + i + " id: " + searchHit.getId(), searchHit.field("min").value(), equalTo((long) i)); + } + + // test the double values + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addScriptField("min", new Script("get min double", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .addSort(SortBuilders.fieldSort("ord").order(SortOrder.ASC).unmappedType("long")) + .setSize(10) + .get(); + + assertNoFailures(searchResponse); + + assertHitCount(searchResponse, 20L); + for (int i = 0; i < 10; i++) { + SearchHit searchHit = searchResponse.getHits().getAt(i); + assertThat("res: " + i + " id: " + searchHit.getId(), searchHit.field("min").value(), equalTo((double) i)); + } + + // test the string values + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addScriptField("min", new Script("get min string", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .addSort(SortBuilders.fieldSort("ord").order(SortOrder.ASC).unmappedType("long")) + .setSize(10) + .get(); + + assertNoFailures(searchResponse); + + assertHitCount(searchResponse, 20L); + for (int i = 0; i < 10; i++) { + SearchHit searchHit = searchResponse.getHits().getAt(i); + assertThat("res: " + i + " id: " + searchHit.getId(), searchHit.field("min").value(), equalTo(i)); + } + + // test the geopoint values + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addScriptField("min", new Script("get min geopoint lon", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .addSort(SortBuilders.fieldSort("ord").order(SortOrder.ASC).unmappedType("long")) + .setSize(10) + .get(); + + assertNoFailures(searchResponse); + + assertHitCount(searchResponse, 20L); + for (int i = 0; i < 10; i++) { + SearchHit searchHit = searchResponse.getHits().getAt(i); + assertThat("res: " + i + " id: " + searchHit.getId(), searchHit.field("min").value(), closeTo(i, GeoUtils.TOLERANCE)); + } + } + + public void testDocumentsWithNullValue() throws Exception { + // TODO: sort shouldn't fail when sort field is mapped dynamically + // We have to specify mapping explicitly because by the time search is performed dynamic mapping might not + // be propagated to all nodes yet and sort operation fail when the sort field is not defined + String mapping = jsonBuilder() + .startObject() + .startObject("type1") + .startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() + .startObject("svalue") + .field("type", "keyword") + .endObject() + .endObject() + .endObject() + .endObject().string(); + assertAcked(prepareCreate("test").addMapping("type1", mapping)); + ensureGreen(); + + client().prepareIndex("test", "type1") + .setSource(jsonBuilder().startObject() + .field("id", "1") + .field("svalue", "aaa") + .endObject()) + .get(); + + client().prepareIndex("test", "type1") + .setSource(jsonBuilder().startObject() + .field("id", "2") + .nullField("svalue") + .endObject()) + .get(); + + client().prepareIndex("test", "type1") + .setSource(jsonBuilder().startObject() + .field("id", "3") + .field("svalue", "bbb") + .endObject()) + .get(); + + flush(); + refresh(); + + Script scripField = new Script("doc['id'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + + SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addScriptField("id", scripField) + .addSort("svalue", SortOrder.ASC) + .get(); + + assertNoFailures(searchResponse); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().getAt(0).field("id").value(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).field("id").value(), equalTo("3")); + assertThat(searchResponse.getHits().getAt(2).field("id").value(), equalTo("2")); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addScriptField("id", new Script("doc['id'].values[0]", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) + .addSort("svalue", SortOrder.ASC) + .get(); + + assertNoFailures(searchResponse); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().getAt(0).field("id").value(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).field("id").value(), equalTo("3")); + assertThat(searchResponse.getHits().getAt(2).field("id").value(), equalTo("2")); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addScriptField("id", scripField) + .addSort("svalue", SortOrder.DESC) + .get(); + + if (searchResponse.getFailedShards() > 0) { + logger.warn("Failed shards:"); + for (ShardSearchFailure shardSearchFailure : searchResponse.getShardFailures()) { + logger.warn("-> {}", shardSearchFailure); + } + } + assertThat(searchResponse.getFailedShards(), equalTo(0)); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().getAt(0).field("id").value(), equalTo("3")); + assertThat(searchResponse.getHits().getAt(1).field("id").value(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(2).field("id").value(), equalTo("2")); + + // a query with docs just with null values + searchResponse = client().prepareSearch() + .setQuery(termQuery("id", "2")) + .addScriptField("id", scripField) + .addSort("svalue", SortOrder.DESC) + .get(); + + if (searchResponse.getFailedShards() > 0) { + logger.warn("Failed shards:"); + for (ShardSearchFailure shardSearchFailure : searchResponse.getShardFailures()) { + logger.warn("-> {}", shardSearchFailure); + } + } + assertThat(searchResponse.getFailedShards(), equalTo(0)); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); + assertThat(searchResponse.getHits().getAt(0).field("id").value(), equalTo("2")); + } + + public void test2920() throws IOException { + assertAcked(prepareCreate("test") + .addMapping("test", jsonBuilder() + .startObject() + .startObject("test") + .startObject("properties") + .startObject("value") + .field("type", "keyword") + .endObject() + .endObject() + .endObject() + .endObject())); + ensureGreen(); + for (int i = 0; i < 10; i++) { + client().prepareIndex("test", "test", Integer.toString(i)) + .setSource(jsonBuilder().startObject().field("value", "" + i).endObject()).get(); + } + refresh(); + + Script sortScript = new Script("\u0027\u0027", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(scriptSort(sortScript, ScriptSortType.STRING)) + .setSize(10) + .get(); + assertNoFailures(searchResponse); + } +} diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java b/core/src/test/java/org/elasticsearch/search/stats/SearchStatsIT.java similarity index 93% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java rename to core/src/test/java/org/elasticsearch/search/stats/SearchStatsIT.java index 52b2f5af797..3162e94f3e2 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java +++ b/core/src/test/java/org/elasticsearch/search/stats/SearchStatsIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.search.stats; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; @@ -32,19 +32,22 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.search.stats.SearchStats.Stats; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; -import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.test.ESIntegTestCase; import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.Map; import java.util.Set; +import java.util.function.Function; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.script.ScriptService.ScriptType; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; @@ -56,13 +59,23 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -/** - */ @ESIntegTestCase.ClusterScope(minNumDataNodes = 2) -public class SearchStatsTests extends ESIntegTestCase { +public class SearchStatsIT extends ESIntegTestCase { + @Override protected Collection<Class<? extends Plugin>> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); + return Collections.singleton(CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends MockScriptPlugin { + + @Override + protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() { + return Collections.singletonMap("_source.field", vars -> { + Map<?, ?> src = (Map) vars.get("_source"); + return src.get("field"); + }); + } } @Override @@ -108,7 +121,7 @@ public class SearchStatsTests extends ESIntegTestCase { SearchResponse searchResponse = internalCluster().coordOnlyNodeClient().prepareSearch() .setQuery(QueryBuilders.termQuery("field", "value")).setStats("group1", "group2") .highlighter(new HighlightBuilder().field("field")) - .addScriptField("scrip1", new Script("_source.field")) + .addScriptField("script1", new Script("_source.field", ScriptType.INLINE, CustomScriptPlugin.NAME, null)) .setSize(100) .execute().actionGet(); assertHitCount(searchResponse, docsTest1 + docsTest2); diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexLookupTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexLookupTests.java deleted file mode 100644 index f8cab2998dc..00000000000 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexLookupTests.java +++ /dev/null @@ -1,629 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.messy.tests; - -import org.elasticsearch.action.search.SearchPhaseExecutionException; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.groovy.GroovyPlugin; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.test.ESIntegTestCase; -import org.hamcrest.Matchers; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; - -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.equalTo; - -public class IndexLookupTests extends ESIntegTestCase { - String includeAllFlag = "_FREQUENCIES | _OFFSETS | _PAYLOADS | _POSITIONS | _CACHE"; - String includeAllWithoutRecordFlag = "_FREQUENCIES | _OFFSETS | _PAYLOADS | _POSITIONS "; - private HashMap<String, List<Object>> expectedEndOffsetsArray; - private HashMap<String, List<Object>> expectedPayloadsArray; - private HashMap<String, List<Object>> expectedPositionsArray; - private HashMap<String, List<Object>> emptyArray; - private HashMap<String, List<Object>> expectedStartOffsetsArray; - - @Override - protected Collection<Class<? extends Plugin>> nodePlugins() { - return Collections.singleton(GroovyPlugin.class); - } - - void initTestData() throws InterruptedException, ExecutionException, IOException { - emptyArray = new HashMap<>(); - List<Object> empty1 = new ArrayList<>(); - empty1.add(-1); - empty1.add(-1); - emptyArray.put("1", empty1); - List<Object> empty2 = new ArrayList<>(); - empty2.add(-1); - empty2.add(-1); - emptyArray.put("2", empty2); - List<Object> empty3 = new ArrayList<>(); - empty3.add(-1); - empty3.add(-1); - emptyArray.put("3", empty3); - - expectedPositionsArray = new HashMap<>(); - - List<Object> pos1 = new ArrayList<>(); - pos1.add(1); - pos1.add(2); - expectedPositionsArray.put("1", pos1); - List<Object> pos2 = new ArrayList<>(); - pos2.add(0); - pos2.add(1); - expectedPositionsArray.put("2", pos2); - List<Object> pos3 = new ArrayList<>(); - pos3.add(0); - pos3.add(4); - expectedPositionsArray.put("3", pos3); - - expectedPayloadsArray = new HashMap<>(); - List<Object> pay1 = new ArrayList<>(); - pay1.add(2); - pay1.add(3); - expectedPayloadsArray.put("1", pay1); - List<Object> pay2 = new ArrayList<>(); - pay2.add(1); - pay2.add(2); - expectedPayloadsArray.put("2", pay2); - List<Object> pay3 = new ArrayList<>(); - pay3.add(1); - pay3.add(-1); - expectedPayloadsArray.put("3", pay3); - /* - * "a|1 b|2 b|3 c|4 d " "b|1 b|2 c|3 d|4 a " "b|1 c|2 d|3 a|4 b " - */ - expectedStartOffsetsArray = new HashMap<>(); - List<Object> starts1 = new ArrayList<>(); - starts1.add(4); - starts1.add(8); - expectedStartOffsetsArray.put("1", starts1); - List<Object> starts2 = new ArrayList<>(); - starts2.add(0); - starts2.add(4); - expectedStartOffsetsArray.put("2", starts2); - List<Object> starts3 = new ArrayList<>(); - starts3.add(0); - starts3.add(16); - expectedStartOffsetsArray.put("3", starts3); - - expectedEndOffsetsArray = new HashMap<>(); - List<Object> ends1 = new ArrayList<>(); - ends1.add(7); - ends1.add(11); - expectedEndOffsetsArray.put("1", ends1); - List<Object> ends2 = new ArrayList<>(); - ends2.add(3); - ends2.add(7); - expectedEndOffsetsArray.put("2", ends2); - List<Object> ends3 = new ArrayList<>(); - ends3.add(3); - ends3.add(17); - expectedEndOffsetsArray.put("3", ends3); - - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") - .startObject("int_payload_field").field("type", "text").field("index_options", "offsets") - .field("analyzer", "payload_int").endObject().endObject().endObject().endObject(); - assertAcked(prepareCreate("test").addMapping("type1", mapping).setSettings( - Settings.builder() - .put(indexSettings()) - .put("index.analysis.analyzer.payload_int.tokenizer", "whitespace") - .putArray("index.analysis.analyzer.payload_int.filter", "delimited_int") - .put("index.analysis.filter.delimited_int.delimiter", "|") - .put("index.analysis.filter.delimited_int.encoding", "int") - .put("index.analysis.filter.delimited_int.type", "delimited_payload_filter"))); - indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("int_payload_field", "a|1 b|2 b|3 c|4 d "), client() - .prepareIndex("test", "type1", "2").setSource("int_payload_field", "b|1 b|2 c|3 d|4 a "), - client().prepareIndex("test", "type1", "3").setSource("int_payload_field", "b|1 c|2 d|3 a|4 b ")); - ensureGreen(); - } - - public void testTwoScripts() throws Exception { - initTestData(); - - // check term frequencies for 'a' - Script scriptFieldScript = new Script("term = _index['int_payload_field']['c']; term.tf()"); - scriptFieldScript = new Script("1"); - Script scoreScript = new Script("term = _index['int_payload_field']['b']; term.tf()"); - Map<String, Object> expectedResultsField = new HashMap<>(); - expectedResultsField.put("1", 1); - expectedResultsField.put("2", 1); - expectedResultsField.put("3", 1); - Map<String, Object> expectedResultsScore = new HashMap<>(); - expectedResultsScore.put("1", 2f); - expectedResultsScore.put("2", 2f); - expectedResultsScore.put("3", 2f); - checkOnlyFunctionScore(scoreScript, expectedResultsScore, 3); - checkValueInEachDocWithFunctionScore(scriptFieldScript, expectedResultsField, scoreScript, expectedResultsScore, 3); - - } - - public void testCallWithDifferentFlagsFails() throws Exception { - initTestData(); - - // should throw an exception, we cannot call with different flags twice - // if the flags of the second call were not included in the first call. - Script script = new Script("term = _index['int_payload_field']['b']; return _index['int_payload_field'].get('b', _POSITIONS).tf();"); - try { - client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()).addScriptField("tvtest", script).execute().actionGet(); - } catch (SearchPhaseExecutionException e) { - assertThat( - "got: " + e.toString(), - e.toString() - .indexOf( - "You must call get with all required flags! Instead of _index['int_payload_field'].get('b', _FREQUENCIES) and _index['int_payload_field'].get('b', _POSITIONS) call _index['int_payload_field'].get('b', _FREQUENCIES | _POSITIONS) once]"), - Matchers.greaterThan(-1)); - } - - // Should not throw an exception this way round - script = new Script( - "term = _index['int_payload_field'].get('b', _POSITIONS | _FREQUENCIES);return _index['int_payload_field']['b'].tf();"); - client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()).addScriptField("tvtest", script).execute().actionGet(); - } - - private void checkOnlyFunctionScore(Script scoreScript, Map<String, Object> expectedScore, int numExpectedDocs) { - SearchResponse sr = client().prepareSearch("test") - .setQuery(QueryBuilders.functionScoreQuery(ScoreFunctionBuilders.scriptFunction(scoreScript))).execute() - .actionGet(); - assertHitCount(sr, numExpectedDocs); - for (SearchHit hit : sr.getHits().getHits()) { - assertThat("for doc " + hit.getId(), ((Float) expectedScore.get(hit.getId())).doubleValue(), - Matchers.closeTo(hit.score(), 1.e-4)); - } - } - - public void testDocumentationExample() throws Exception { - initTestData(); - - Script script = new Script("term = _index['float_payload_field'].get('b'," + includeAllFlag - + "); payloadSum=0; for (pos in term) {payloadSum = pos.payloadAsInt(0)}; payloadSum"); - - // non existing field: sum should be 0 - HashMap<String, Object> zeroArray = new HashMap<>(); - zeroArray.put("1", 0); - zeroArray.put("2", 0); - zeroArray.put("3", 0); - checkValueInEachDoc(script, zeroArray, 3); - - script = new Script("term = _index['int_payload_field'].get('b'," + includeAllFlag - + "); payloadSum=0; for (pos in term) {payloadSum = payloadSum + pos.payloadAsInt(0)}; payloadSum"); - - // existing field: sums should be as here: - zeroArray.put("1", 5); - zeroArray.put("2", 3); - zeroArray.put("3", 1); - checkValueInEachDoc(script, zeroArray, 3); - } - - public void testIteratorAndRecording() throws Exception { - initTestData(); - - // call twice with record: should work as expected - Script script = createPositionsArrayScriptIterateTwice("b", includeAllFlag, "position"); - checkArrayValsInEachDoc(script, expectedPositionsArray, 3); - script = createPositionsArrayScriptIterateTwice("b", includeAllFlag, "startOffset"); - checkArrayValsInEachDoc(script, expectedStartOffsetsArray, 3); - script = createPositionsArrayScriptIterateTwice("b", includeAllFlag, "endOffset"); - checkArrayValsInEachDoc(script, expectedEndOffsetsArray, 3); - script = createPositionsArrayScriptIterateTwice("b", includeAllFlag, "payloadAsInt(-1)"); - checkArrayValsInEachDoc(script, expectedPayloadsArray, 3); - - // no record and get iterator twice: should fail - script = createPositionsArrayScriptIterateTwice("b", includeAllWithoutRecordFlag, "position"); - checkExceptions(script); - script = createPositionsArrayScriptIterateTwice("b", includeAllWithoutRecordFlag, "startOffset"); - checkExceptions(script); - script = createPositionsArrayScriptIterateTwice("b", includeAllWithoutRecordFlag, "endOffset"); - checkExceptions(script); - script = createPositionsArrayScriptIterateTwice("b", includeAllWithoutRecordFlag, "payloadAsInt(-1)"); - checkExceptions(script); - - // no record and get termObject twice and iterate: should fail - script = createPositionsArrayScriptGetInfoObjectTwice("b", includeAllWithoutRecordFlag, "position"); - checkExceptions(script); - script = createPositionsArrayScriptGetInfoObjectTwice("b", includeAllWithoutRecordFlag, "startOffset"); - checkExceptions(script); - script = createPositionsArrayScriptGetInfoObjectTwice("b", includeAllWithoutRecordFlag, "endOffset"); - checkExceptions(script); - script = createPositionsArrayScriptGetInfoObjectTwice("b", includeAllWithoutRecordFlag, "payloadAsInt(-1)"); - checkExceptions(script); - - } - - private Script createPositionsArrayScriptGetInfoObjectTwice(String term, String flags, String what) { - String script = "term = _index['int_payload_field'].get('" + term + "'," + flags - + "); array=[]; for (pos in term) {array.add(pos." + what + ")}; _index['int_payload_field'].get('" + term + "'," - + flags + "); array=[]; for (pos in term) {array.add(pos." + what + ")}"; - return new Script(script); - } - - private Script createPositionsArrayScriptIterateTwice(String term, String flags, String what) { - String script = "term = _index['int_payload_field'].get('" + term + "'," + flags - + "); array=[]; for (pos in term) {array.add(pos." + what + ")}; array=[]; for (pos in term) {array.add(pos." + what - + ")}; array"; - return new Script(script); - } - - private Script createPositionsArrayScript(String field, String term, String flags, String what) { - String script = "term = _index['" + field + "'].get('" + term + "'," + flags - + "); array=[]; for (pos in term) {array.add(pos." + what + ")}; array"; - return new Script(script); - } - - private Script createPositionsArrayScriptDefaultGet(String field, String term, String what) { - String script = "term = _index['" + field + "']['" + term + "']; array=[]; for (pos in term) {array.add(pos." + what - + ")}; array"; - return new Script(script); - } - - public void testFlags() throws Exception { - initTestData(); - - // check default flag - Script script = createPositionsArrayScriptDefaultGet("int_payload_field", "b", "position"); - // there should be no positions - /* TODO: the following tests fail with the new postings enum apis because of a bogus assert in BlockDocsEnum - checkArrayValsInEachDoc(script, emptyArray, 3); - script = createPositionsArrayScriptDefaultGet("int_payload_field", "b", "startOffset"); - // there should be no offsets - checkArrayValsInEachDoc(script, emptyArray, 3); - script = createPositionsArrayScriptDefaultGet("int_payload_field", "b", "endOffset"); - // there should be no offsets - checkArrayValsInEachDoc(script, emptyArray, 3); - script = createPositionsArrayScriptDefaultGet("int_payload_field", "b", "payloadAsInt(-1)"); - // there should be no payload - checkArrayValsInEachDoc(script, emptyArray, 3); - - // check FLAG_FREQUENCIES flag - script = createPositionsArrayScript("int_payload_field", "b", "_FREQUENCIES", "position"); - // there should be no positions - checkArrayValsInEachDoc(script, emptyArray, 3); - script = createPositionsArrayScript("int_payload_field", "b", "_FREQUENCIES", "startOffset"); - // there should be no offsets - checkArrayValsInEachDoc(script, emptyArray, 3); - script = createPositionsArrayScript("int_payload_field", "b", "_FREQUENCIES", "endOffset"); - // there should be no offsets - checkArrayValsInEachDoc(script, emptyArray, 3); - script = createPositionsArrayScript("int_payload_field", "b", "_FREQUENCIES", "payloadAsInt(-1)"); - // there should be no payloads - checkArrayValsInEachDoc(script, emptyArray, 3);*/ - - // check FLAG_POSITIONS flag - script = createPositionsArrayScript("int_payload_field", "b", "_POSITIONS", "position"); - // there should be positions - checkArrayValsInEachDoc(script, expectedPositionsArray, 3); - /* TODO: these tests make a bogus assumption that asking for positions will return only positions - script = createPositionsArrayScript("int_payload_field", "b", "_POSITIONS", "startOffset"); - // there should be no offsets - checkArrayValsInEachDoc(script, emptyArray, 3); - script = createPositionsArrayScript("int_payload_field", "b", "_POSITIONS", "endOffset"); - // there should be no offsets - checkArrayValsInEachDoc(script, emptyArray, 3); - script = createPositionsArrayScript("int_payload_field", "b", "_POSITIONS", "payloadAsInt(-1)"); - // there should be no payloads - checkArrayValsInEachDoc(script, emptyArray, 3);*/ - - // check FLAG_OFFSETS flag - script = createPositionsArrayScript("int_payload_field", "b", "_OFFSETS", "position"); - // there should be positions and s forth ... - checkArrayValsInEachDoc(script, expectedPositionsArray, 3); - script = createPositionsArrayScript("int_payload_field", "b", "_OFFSETS", "startOffset"); - checkArrayValsInEachDoc(script, expectedStartOffsetsArray, 3); - script = createPositionsArrayScript("int_payload_field", "b", "_OFFSETS", "endOffset"); - checkArrayValsInEachDoc(script, expectedEndOffsetsArray, 3); - script = createPositionsArrayScript("int_payload_field", "b", "_OFFSETS", "payloadAsInt(-1)"); - checkArrayValsInEachDoc(script, expectedPayloadsArray, 3); - - // check FLAG_PAYLOADS flag - script = createPositionsArrayScript("int_payload_field", "b", "_PAYLOADS", "position"); - checkArrayValsInEachDoc(script, expectedPositionsArray, 3); - script = createPositionsArrayScript("int_payload_field", "b", "_PAYLOADS", "startOffset"); - checkArrayValsInEachDoc(script, expectedStartOffsetsArray, 3); - script = createPositionsArrayScript("int_payload_field", "b", "_PAYLOADS", "endOffset"); - checkArrayValsInEachDoc(script, expectedEndOffsetsArray, 3); - script = createPositionsArrayScript("int_payload_field", "b", "_PAYLOADS", "payloadAsInt(-1)"); - checkArrayValsInEachDoc(script, expectedPayloadsArray, 3); - - // check all flags - String allFlags = "_POSITIONS | _OFFSETS | _PAYLOADS"; - script = createPositionsArrayScript("int_payload_field", "b", allFlags, "position"); - checkArrayValsInEachDoc(script, expectedPositionsArray, 3); - script = createPositionsArrayScript("int_payload_field", "b", allFlags, "startOffset"); - checkArrayValsInEachDoc(script, expectedStartOffsetsArray, 3); - script = createPositionsArrayScript("int_payload_field", "b", allFlags, "endOffset"); - checkArrayValsInEachDoc(script, expectedEndOffsetsArray, 3); - script = createPositionsArrayScript("int_payload_field", "b", allFlags, "payloadAsInt(-1)"); - checkArrayValsInEachDoc(script, expectedPayloadsArray, 3); - - // check all flags without record - script = createPositionsArrayScript("int_payload_field", "b", includeAllWithoutRecordFlag, "position"); - checkArrayValsInEachDoc(script, expectedPositionsArray, 3); - script = createPositionsArrayScript("int_payload_field", "b", includeAllWithoutRecordFlag, "startOffset"); - checkArrayValsInEachDoc(script, expectedStartOffsetsArray, 3); - script = createPositionsArrayScript("int_payload_field", "b", includeAllWithoutRecordFlag, "endOffset"); - checkArrayValsInEachDoc(script, expectedEndOffsetsArray, 3); - script = createPositionsArrayScript("int_payload_field", "b", includeAllWithoutRecordFlag, "payloadAsInt(-1)"); - checkArrayValsInEachDoc(script, expectedPayloadsArray, 3); - - } - - private void checkArrayValsInEachDoc(Script script, HashMap<String, List<Object>> expectedArray, int expectedHitSize) { - SearchResponse sr = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()).addScriptField("tvtest", script) - .execute().actionGet(); - assertHitCount(sr, expectedHitSize); - int nullCounter = 0; - for (SearchHit hit : sr.getHits().getHits()) { - Object result = hit.getFields().get("tvtest").getValues(); - Object expectedResult = expectedArray.get(hit.getId()); - assertThat("for doc " + hit.getId(), result, equalTo(expectedResult)); - if (expectedResult != null) { - nullCounter++; - } - } - assertThat(nullCounter, equalTo(expectedArray.size())); - } - - public void testAllExceptPosAndOffset() throws Exception { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") - .startObject("float_payload_field").field("type", "text").field("index_options", "offsets").field("term_vector", "no") - .field("analyzer", "payload_float").endObject().startObject("string_payload_field").field("type", "text") - .field("index_options", "offsets").field("term_vector", "no").field("analyzer", "payload_string").endObject() - .startObject("int_payload_field").field("type", "text").field("index_options", "offsets") - .field("analyzer", "payload_int").endObject().endObject().endObject().endObject(); - assertAcked(prepareCreate("test").addMapping("type1", mapping).setSettings( - Settings.builder() - .put(indexSettings()) - .put("index.analysis.analyzer.payload_float.tokenizer", "whitespace") - .putArray("index.analysis.analyzer.payload_float.filter", "delimited_float") - .put("index.analysis.filter.delimited_float.delimiter", "|") - .put("index.analysis.filter.delimited_float.encoding", "float") - .put("index.analysis.filter.delimited_float.type", "delimited_payload_filter") - .put("index.analysis.analyzer.payload_string.tokenizer", "whitespace") - .putArray("index.analysis.analyzer.payload_string.filter", "delimited_string") - .put("index.analysis.filter.delimited_string.delimiter", "|") - .put("index.analysis.filter.delimited_string.encoding", "identity") - .put("index.analysis.filter.delimited_string.type", "delimited_payload_filter") - .put("index.analysis.analyzer.payload_int.tokenizer", "whitespace") - .putArray("index.analysis.analyzer.payload_int.filter", "delimited_int") - .put("index.analysis.filter.delimited_int.delimiter", "|") - .put("index.analysis.filter.delimited_int.encoding", "int") - .put("index.analysis.filter.delimited_int.type", "delimited_payload_filter") - .put("index.number_of_shards", 1))); - indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("float_payload_field", "a|1 b|2 a|3 b "), client() - .prepareIndex("test", "type1", "2").setSource("string_payload_field", "a|a b|b a|a b "), - client().prepareIndex("test", "type1", "3").setSource("float_payload_field", "a|4 b|5 a|6 b "), - client().prepareIndex("test", "type1", "4").setSource("string_payload_field", "a|b b|a a|b b "), - client().prepareIndex("test", "type1", "5").setSource("float_payload_field", "c "), - client().prepareIndex("test", "type1", "6").setSource("int_payload_field", "c|1")); - - // get the number of all docs - Script script = new Script("_index.numDocs()"); - checkValueInEachDoc(6, script, 6); - - // get the number of docs with field float_payload_field - script = new Script("_index['float_payload_field'].docCount()"); - checkValueInEachDoc(3, script, 6); - - // corner case: what if the field does not exist? - script = new Script("_index['non_existent_field'].docCount()"); - checkValueInEachDoc(0, script, 6); - - // get the number of all tokens in all docs - script = new Script("_index['float_payload_field'].sumttf()"); - checkValueInEachDoc(9, script, 6); - - // corner case get the number of all tokens in all docs for non existent - // field - script = new Script("_index['non_existent_field'].sumttf()"); - checkValueInEachDoc(0, script, 6); - - // get the sum of doc freqs in all docs - script = new Script("_index['float_payload_field'].sumdf()"); - checkValueInEachDoc(5, script, 6); - - // get the sum of doc freqs in all docs for non existent field - script = new Script("_index['non_existent_field'].sumdf()"); - checkValueInEachDoc(0, script, 6); - - // check term frequencies for 'a' - script = new Script("term = _index['float_payload_field']['a']; if (term != null) {term.tf()}"); - Map<String, Object> expectedResults = new HashMap<>(); - expectedResults.put("1", 2); - expectedResults.put("2", 0); - expectedResults.put("3", 2); - expectedResults.put("4", 0); - expectedResults.put("5", 0); - expectedResults.put("6", 0); - checkValueInEachDoc(script, expectedResults, 6); - expectedResults.clear(); - - // check doc frequencies for 'c' - script = new Script("term = _index['float_payload_field']['c']; if (term != null) {term.df()}"); - expectedResults.put("1", 1L); - expectedResults.put("2", 1L); - expectedResults.put("3", 1L); - expectedResults.put("4", 1L); - expectedResults.put("5", 1L); - expectedResults.put("6", 1L); - checkValueInEachDoc(script, expectedResults, 6); - expectedResults.clear(); - - // check doc frequencies for term that does not exist - script = new Script("term = _index['float_payload_field']['non_existent_term']; if (term != null) {term.df()}"); - expectedResults.put("1", 0L); - expectedResults.put("2", 0L); - expectedResults.put("3", 0L); - expectedResults.put("4", 0L); - expectedResults.put("5", 0L); - expectedResults.put("6", 0L); - checkValueInEachDoc(script, expectedResults, 6); - expectedResults.clear(); - - // check doc frequencies for term that does not exist - script = new Script("term = _index['non_existent_field']['non_existent_term']; if (term != null) {term.tf()}"); - expectedResults.put("1", 0); - expectedResults.put("2", 0); - expectedResults.put("3", 0); - expectedResults.put("4", 0); - expectedResults.put("5", 0); - expectedResults.put("6", 0); - checkValueInEachDoc(script, expectedResults, 6); - expectedResults.clear(); - - // check total term frequencies for 'a' - script = new Script("term = _index['float_payload_field']['a']; if (term != null) {term.ttf()}"); - expectedResults.put("1", 4L); - expectedResults.put("2", 4L); - expectedResults.put("3", 4L); - expectedResults.put("4", 4L); - expectedResults.put("5", 4L); - expectedResults.put("6", 4L); - checkValueInEachDoc(script, expectedResults, 6); - expectedResults.clear(); - - // check float payload for 'b' - HashMap<String, List<Object>> expectedPayloadsArray = new HashMap<>(); - script = createPositionsArrayScript("float_payload_field", "b", includeAllFlag, "payloadAsFloat(-1)"); - float missingValue = -1; - List<Object> payloadsFor1 = new ArrayList<>(); - payloadsFor1.add(2f); - payloadsFor1.add(missingValue); - expectedPayloadsArray.put("1", payloadsFor1); - List<Object> payloadsFor2 = new ArrayList<>(); - payloadsFor2.add(5f); - payloadsFor2.add(missingValue); - expectedPayloadsArray.put("3", payloadsFor2); - expectedPayloadsArray.put("6", new ArrayList<>()); - expectedPayloadsArray.put("5", new ArrayList<>()); - expectedPayloadsArray.put("4", new ArrayList<>()); - expectedPayloadsArray.put("2", new ArrayList<>()); - checkArrayValsInEachDoc(script, expectedPayloadsArray, 6); - - // check string payload for 'b' - expectedPayloadsArray.clear(); - payloadsFor1.clear(); - payloadsFor2.clear(); - script = createPositionsArrayScript("string_payload_field", "b", includeAllFlag, "payloadAsString()"); - payloadsFor1.add("b"); - payloadsFor1.add(null); - expectedPayloadsArray.put("2", payloadsFor1); - payloadsFor2.add("a"); - payloadsFor2.add(null); - expectedPayloadsArray.put("4", payloadsFor2); - expectedPayloadsArray.put("6", new ArrayList<>()); - expectedPayloadsArray.put("5", new ArrayList<>()); - expectedPayloadsArray.put("3", new ArrayList<>()); - expectedPayloadsArray.put("1", new ArrayList<>()); - checkArrayValsInEachDoc(script, expectedPayloadsArray, 6); - - // check int payload for 'c' - expectedPayloadsArray.clear(); - payloadsFor1.clear(); - payloadsFor2.clear(); - script = createPositionsArrayScript("int_payload_field", "c", includeAllFlag, "payloadAsInt(-1)"); - payloadsFor1 = new ArrayList<>(); - payloadsFor1.add(1); - expectedPayloadsArray.put("6", payloadsFor1); - expectedPayloadsArray.put("5", new ArrayList<>()); - expectedPayloadsArray.put("4", new ArrayList<>()); - expectedPayloadsArray.put("3", new ArrayList<>()); - expectedPayloadsArray.put("2", new ArrayList<>()); - expectedPayloadsArray.put("1", new ArrayList<>()); - checkArrayValsInEachDoc(script, expectedPayloadsArray, 6); - - } - - private void checkExceptions(Script script) { - try { - SearchResponse sr = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()).addScriptField("tvtest", script) - .execute().actionGet(); - assertThat(sr.getHits().hits().length, equalTo(0)); - ShardSearchFailure[] shardFails = sr.getShardFailures(); - for (ShardSearchFailure fail : shardFails) { - assertThat(fail.reason().indexOf("Cannot iterate twice! If you want to iterate more that once, add _CACHE explicitly."), - Matchers.greaterThan(-1)); - } - } catch (SearchPhaseExecutionException ex) { - assertThat( - "got " + ex.toString(), - ex.toString().indexOf("Cannot iterate twice! If you want to iterate more that once, add _CACHE explicitly."), - Matchers.greaterThan(-1)); - } - } - - private void checkValueInEachDocWithFunctionScore(Script fieldScript, Map<String, Object> expectedFieldVals, Script scoreScript, - Map<String, Object> expectedScore, int numExpectedDocs) { - SearchResponse sr = client().prepareSearch("test") - .setQuery(QueryBuilders.functionScoreQuery(ScoreFunctionBuilders.scriptFunction(scoreScript))) - .addScriptField("tvtest", fieldScript).execute().actionGet(); - assertHitCount(sr, numExpectedDocs); - for (SearchHit hit : sr.getHits().getHits()) { - Object result = hit.getFields().get("tvtest").getValues().get(0); - Object expectedResult = expectedFieldVals.get(hit.getId()); - assertThat("for doc " + hit.getId(), result, equalTo(expectedResult)); - assertThat("for doc " + hit.getId(), ((Float) expectedScore.get(hit.getId())).doubleValue(), - Matchers.closeTo(hit.score(), 1.e-4)); - } - } - - private void checkValueInEachDoc(Script script, Map<String, Object> expectedResults, int numExpectedDocs) { - SearchResponse sr = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()).addScriptField("tvtest", script) - .execute().actionGet(); - assertHitCount(sr, numExpectedDocs); - for (SearchHit hit : sr.getHits().getHits()) { - Object result = hit.getFields().get("tvtest").getValues().get(0); - Object expectedResult = expectedResults.get(hit.getId()); - assertThat("for doc " + hit.getId(), result, equalTo(expectedResult)); - } - } - - private void checkValueInEachDoc(int value, Script script, int numExpectedDocs) { - SearchResponse sr = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()).addScriptField("tvtest", script) - .execute().actionGet(); - assertHitCount(sr, numExpectedDocs); - for (SearchHit hit : sr.getHits().getHits()) { - Object result = hit.getFields().get("tvtest").getValues().get(0); - if (result instanceof Integer) { - assertThat((Integer)result, equalTo(value)); - } else if (result instanceof Long) { - assertThat(((Long) result).intValue(), equalTo(value)); - } else { - fail(); - } - } - } -} diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java deleted file mode 100644 index 78d95edecff..00000000000 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java +++ /dev/null @@ -1,334 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.messy.tests; - - -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; -import org.elasticsearch.common.geo.GeoUtils; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.groovy.GroovyPlugin; -import org.elasticsearch.search.sort.ScriptSortBuilder; -import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType; -import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Random; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.hamcrest.Matchers.closeTo; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; - -/** - * - */ -public class SimpleSortTests extends ESIntegTestCase { - @Override - protected Collection<Class<? extends Plugin>> nodePlugins() { - return pluginList(GroovyPlugin.class, InternalSettingsPlugin.class); - } - - public void testSimpleSorts() throws Exception { - Random random = random(); - assertAcked(prepareCreate("test") - .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") - .startObject("str_value").field("type", "keyword").endObject() - .startObject("boolean_value").field("type", "boolean").endObject() - .startObject("byte_value").field("type", "byte").endObject() - .startObject("short_value").field("type", "short").endObject() - .startObject("integer_value").field("type", "integer").endObject() - .startObject("long_value").field("type", "long").endObject() - .startObject("float_value").field("type", "float").endObject() - .startObject("double_value").field("type", "double").endObject() - .endObject().endObject().endObject())); - ensureGreen(); - List<IndexRequestBuilder> builders = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - IndexRequestBuilder builder = client().prepareIndex("test", "type1", Integer.toString(i)).setSource(jsonBuilder().startObject() - .field("str_value", new String(new char[]{(char) (97 + i), (char) (97 + i)})) - .field("boolean_value", true) - .field("byte_value", i) - .field("short_value", i) - .field("integer_value", i) - .field("long_value", i) - .field("float_value", 0.1 * i) - .field("double_value", 0.1 * i) - .endObject()); - builders.add(builder); - } - Collections.shuffle(builders, random); - for (IndexRequestBuilder builder : builders) { - builder.execute().actionGet(); - if (random.nextBoolean()) { - if (random.nextInt(5) != 0) { - refresh(); - } else { - client().admin().indices().prepareFlush().execute().actionGet(); - } - } - - } - refresh(); - - // STRING script - int size = 1 + random.nextInt(10); - - SearchResponse searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(size) - .addSort(new ScriptSortBuilder(new Script("doc['str_value'].value"), ScriptSortType.STRING)).execute().actionGet(); - assertHitCount(searchResponse, 10); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - for (int i = 0; i < size; i++) { - assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(i))); - assertThat(searchResponse.getHits().getAt(i).sortValues()[0].toString(), equalTo(new String(new char[] { (char) (97 + i), - (char) (97 + i) }))); - } - size = 1 + random.nextInt(10); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("str_value", SortOrder.DESC).execute() - .actionGet(); - - assertHitCount(searchResponse, 10); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - for (int i = 0; i < size; i++) { - assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(9 - i))); - assertThat(searchResponse.getHits().getAt(i).sortValues()[0].toString(), equalTo(new String(new char[] { (char) (97 + (9 - i)), - (char) (97 + (9 - i)) }))); - } - - assertThat(searchResponse.toString(), not(containsString("error"))); - - assertNoFailures(searchResponse); - } - - public void testSortMinValueScript() throws IOException { - String mapping = jsonBuilder().startObject().startObject("type1").startObject("properties") - .startObject("lvalue").field("type", "long").endObject() - .startObject("dvalue").field("type", "double").endObject() - .startObject("svalue").field("type", "keyword").endObject() - .startObject("gvalue").field("type", "geo_point").endObject() - .endObject().endObject().endObject().string(); - assertAcked(prepareCreate("test").addMapping("type1", mapping)); - ensureGreen(); - - for (int i = 0; i < 10; i++) { - IndexRequestBuilder req = client().prepareIndex("test", "type1", "" + i).setSource(jsonBuilder().startObject() - .field("ord", i) - .field("svalue", new String[]{"" + i, "" + (i + 1), "" + (i + 2)}) - .field("lvalue", new long[]{i, i + 1, i + 2}) - .field("dvalue", new double[]{i, i + 1, i + 2}) - .startObject("gvalue") - .field("lat", (double) i + 1) - .field("lon", (double) i) - .endObject() - .endObject()); - req.execute().actionGet(); - } - - for (int i = 10; i < 20; i++) { // add some docs that don't have values in those fields - client().prepareIndex("test", "type1", "" + i).setSource(jsonBuilder().startObject() - .field("ord", i) - .endObject()).execute().actionGet(); - } - client().admin().indices().prepareRefresh("test").execute().actionGet(); - - // test the long values - SearchResponse searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addScriptField("min", new Script("retval = Long.MAX_VALUE; for (v in doc['lvalue'].values){ retval = min(v, retval) }; retval")) - .addSort(SortBuilders.fieldSort("ord").order(SortOrder.ASC).unmappedType("long")).setSize(10) - .execute().actionGet(); - - assertNoFailures(searchResponse); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(20L)); - for (int i = 0; i < 10; i++) { - assertThat("res: " + i + " id: " + searchResponse.getHits().getAt(i).getId(), (Long) searchResponse.getHits().getAt(i).field("min").value(), equalTo((long) i)); - } - // test the double values - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addScriptField("min", new Script("retval = Double.MAX_VALUE; for (v in doc['dvalue'].values){ retval = min(v, retval) }; retval")) - .addSort(SortBuilders.fieldSort("ord").order(SortOrder.ASC).unmappedType("long")).setSize(10) - .execute().actionGet(); - - assertNoFailures(searchResponse); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(20L)); - for (int i = 0; i < 10; i++) { - assertThat("res: " + i + " id: " + searchResponse.getHits().getAt(i).getId(), (Double) searchResponse.getHits().getAt(i).field("min").value(), equalTo((double) i)); - } - - // test the string values - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addScriptField("min", new Script("retval = Integer.MAX_VALUE; for (v in doc['svalue'].values){ retval = min(Integer.parseInt(v), retval) }; retval")) - .addSort(SortBuilders.fieldSort("ord").order(SortOrder.ASC).unmappedType("long")).setSize(10) - .execute().actionGet(); - - assertNoFailures(searchResponse); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(20L)); - for (int i = 0; i < 10; i++) { - assertThat("res: " + i + " id: " + searchResponse.getHits().getAt(i).getId(), (Integer) searchResponse.getHits().getAt(i).field("min").value(), equalTo(i)); - } - - // test the geopoint values - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addScriptField("min", new Script("retval = Double.MAX_VALUE; for (v in doc['gvalue'].values){ retval = min(v.lon, retval) }; retval")) - .addSort(SortBuilders.fieldSort("ord").order(SortOrder.ASC).unmappedType("long")).setSize(10) - .execute().actionGet(); - - assertNoFailures(searchResponse); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(20L)); - for (int i = 0; i < 10; i++) { - assertThat("res: " + i + " id: " + searchResponse.getHits().getAt(i).getId(), (Double) searchResponse.getHits().getAt(i).field("min").value(), closeTo(i, GeoUtils.TOLERANCE)); - } - } - - public void testDocumentsWithNullValue() throws Exception { - // TODO: sort shouldn't fail when sort field is mapped dynamically - // We have to specify mapping explicitly because by the time search is performed dynamic mapping might not - // be propagated to all nodes yet and sort operation fail when the sort field is not defined - String mapping = jsonBuilder().startObject().startObject("type1").startObject("properties") - .startObject("id").field("type", "keyword").endObject() - .startObject("svalue").field("type", "keyword").endObject() - .endObject().endObject().endObject().string(); - assertAcked(prepareCreate("test").addMapping("type1", mapping)); - ensureGreen(); - - client().prepareIndex("test", "type1").setSource(jsonBuilder().startObject() - .field("id", "1") - .field("svalue", "aaa") - .endObject()).execute().actionGet(); - - client().prepareIndex("test", "type1").setSource(jsonBuilder().startObject() - .field("id", "2") - .nullField("svalue") - .endObject()).execute().actionGet(); - - client().prepareIndex("test", "type1").setSource(jsonBuilder().startObject() - .field("id", "3") - .field("svalue", "bbb") - .endObject()).execute().actionGet(); - - - flush(); - refresh(); - - SearchResponse searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addScriptField("id", new Script("doc['id'].value")) - .addSort("svalue", SortOrder.ASC) - .execute().actionGet(); - - assertNoFailures(searchResponse); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().getAt(0).field("id").value(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(1).field("id").value(), equalTo("3")); - assertThat(searchResponse.getHits().getAt(2).field("id").value(), equalTo("2")); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addScriptField("id", new Script("doc['id'].values[0]")) - .addSort("svalue", SortOrder.ASC) - .execute().actionGet(); - - assertNoFailures(searchResponse); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().getAt(0).field("id").value(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(1).field("id").value(), equalTo("3")); - assertThat(searchResponse.getHits().getAt(2).field("id").value(), equalTo("2")); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addScriptField("id", new Script("doc['id'].value")) - .addSort("svalue", SortOrder.DESC) - .execute().actionGet(); - - if (searchResponse.getFailedShards() > 0) { - logger.warn("Failed shards:"); - for (ShardSearchFailure shardSearchFailure : searchResponse.getShardFailures()) { - logger.warn("-> {}", shardSearchFailure); - } - } - assertThat(searchResponse.getFailedShards(), equalTo(0)); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().getAt(0).field("id").value(), equalTo("3")); - assertThat(searchResponse.getHits().getAt(1).field("id").value(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(2).field("id").value(), equalTo("2")); - - // a query with docs just with null values - searchResponse = client().prepareSearch() - .setQuery(termQuery("id", "2")) - .addScriptField("id", new Script("doc['id'].value")) - .addSort("svalue", SortOrder.DESC) - .execute().actionGet(); - - if (searchResponse.getFailedShards() > 0) { - logger.warn("Failed shards:"); - for (ShardSearchFailure shardSearchFailure : searchResponse.getShardFailures()) { - logger.warn("-> {}", shardSearchFailure); - } - } - assertThat(searchResponse.getFailedShards(), equalTo(0)); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); - assertThat(searchResponse.getHits().getAt(0).field("id").value(), equalTo("2")); - } - - public void test2920() throws IOException { - assertAcked(prepareCreate("test").addMapping( - "test", - jsonBuilder().startObject().startObject("test").startObject("properties").startObject("value").field("type", "keyword") - .endObject().endObject().endObject().endObject())); - ensureGreen(); - for (int i = 0; i < 10; i++) { - client().prepareIndex("test", "test", Integer.toString(i)) - .setSource(jsonBuilder().startObject().field("value", "" + i).endObject()).execute().actionGet(); - } - refresh(); - SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()) - .addSort(SortBuilders.scriptSort(new Script("\u0027\u0027"), ScriptSortType.STRING)).setSize(10).execute().actionGet(); - assertNoFailures(searchResponse); - } -} diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java deleted file mode 100644 index 383e7dd6028..00000000000 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** - * This package contains tests that use groovy to test what looks - * to be unrelated functionality, or functionality that should be - * tested with a mock instead. Instead of doing an epic battle - * with these tests, they are temporarily moved here to the groovy - * plugin's tests, but that is likely not where they belong. Please - * help by cleaning them up and we can remove this package! - * - * <ul> - * <li>If the test is actually testing groovy specifically, move to - * the org.elasticsearch.script.groovy tests package of this plugin</li> - * <li>If the test is testing scripting integration with another core subsystem, - * fix it to use a mock instead, so it can be in the core tests again</li> - * <li>If the test is just being lazy, and does not really need scripting to test - * something, clean it up!</li> - * </ul> - */ -/* List of renames that took place: - renamed: core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/AvgTests.java - renamed: core/src/test/java/org/elasticsearch/document/BulkIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java - renamed: core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ChildQuerySearchTests.java - renamed: core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java - ^^^^^ note: the methods from this test using mustache were moved to the mustache module under its messy tests package. - renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateHistogramTests.java - renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateRangeTests.java - renamed: core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/FunctionScoreTests.java - renamed: core/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java - renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/IPv4RangeIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java - renamed: core/src/test/java/org/elasticsearch/script/IndexLookupIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexLookupTests.java - renamed: core/src/test/java/org/elasticsearch/script/IndexedScriptIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexedScriptTests.java - renamed: core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/InnerHitsTests.java - renamed: core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/PercolatorTests.java - renamed: core/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java - renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RangeTests.java - renamed: core/src/test/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java - renamed: core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptedMetricTests.java - renamed: core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java - renamed: core/src/test/java/org/elasticsearch/search/stats/SearchStatsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java - renamed: core/src/test/java/org/elasticsearch/search/timeout/SearchTimeoutIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchTimeoutTests.java - renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SignificantTermsSignificanceScoreTests.java - renamed: core/src/test/java/org/elasticsearch/nested/SimpleNestedIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleNestedTests.java - renamed: core/src/test/java/org/elasticsearch/search/sort/SimpleSortIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java - renamed: core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SumTests.java - renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/TopHitsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TopHitsTests.java - renamed: core/src/test/java/org/elasticsearch/index/mapper/TransformOnIndexMapperIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TransformOnIndexMapperTests.java - renamed: core/src/main/java/org/elasticsearch/script/groovy/GroovyScriptCompilationException.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyRestIT.java - renamed: core/src/test/java/org/elasticsearch/script/GroovyScriptIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java - renamed: core/src/test/java/org/elasticsearch/script/GroovySecurityIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java - renamed: core/src/test/resources/org/elasticsearch/search/aggregations/metrics/scripted/conf/scripts/combine_script.groovy -> plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/combine_script.groovy - renamed: core/src/test/resources/org/elasticsearch/search/aggregations/metrics/scripted/conf/scripts/init_script.groovy -> plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/init_script.groovy - renamed: core/src/test/resources/org/elasticsearch/search/aggregations/metrics/scripted/conf/scripts/map_script.groovy -> plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/map_script.groovy - renamed: core/src/test/resources/org/elasticsearch/search/aggregations/metrics/scripted/conf/scripts/reduce_script.groovy -> plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/reduce_script.groovy - renamed: core/src/test/resources/org/elasticsearch/search/aggregations/bucket/config/scripts/significance_script_no_params.groovy -> plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_no_params.groovy - renamed: core/src/test/resources/org/elasticsearch/search/aggregations/bucket/config/scripts/significance_script_with_params.groovy -> plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_with_params.groovy - */ -package org.elasticsearch.messy.tests; diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexedScriptTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyIndexedScriptTests.java similarity index 98% rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexedScriptTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyIndexedScriptTests.java index 623d2cf155c..be307c690f9 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexedScriptTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyIndexedScriptTests.java @@ -18,7 +18,7 @@ */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.script.groovy; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -51,7 +51,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; -public class IndexedScriptTests extends ESIntegTestCase { +public class GroovyIndexedScriptTests extends ESIntegTestCase { @Override protected Collection<Class<? extends Plugin>> nodePlugins() { return Collections.singleton(GroovyPlugin.class); diff --git a/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/combine_script.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/combine_script.groovy deleted file mode 100644 index da13f608757..00000000000 --- a/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/combine_script.groovy +++ /dev/null @@ -1 +0,0 @@ -newaggregation = []; sum = 0;for (a in _agg) { sum += a}; newaggregation.add(sum); return newaggregation \ No newline at end of file diff --git a/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/init_script.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/init_script.groovy deleted file mode 100644 index 6cf4f40b6d2..00000000000 --- a/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/init_script.groovy +++ /dev/null @@ -1 +0,0 @@ -vars.multiplier = 3 \ No newline at end of file diff --git a/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/map_script.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/map_script.groovy deleted file mode 100644 index aece1a7d84d..00000000000 --- a/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/map_script.groovy +++ /dev/null @@ -1 +0,0 @@ -_agg.add(vars.multiplier) \ No newline at end of file diff --git a/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/reduce_script.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/reduce_script.groovy deleted file mode 100644 index 835dcfbb3a8..00000000000 --- a/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/reduce_script.groovy +++ /dev/null @@ -1 +0,0 @@ -newaggregation = []; sum = 0;for (aggregation in _aggs) { for (a in aggregation) { sum += a} }; newaggregation.add(sum); return newaggregation \ No newline at end of file diff --git a/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_no_params.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_no_params.groovy deleted file mode 100644 index 7178e05efbd..00000000000 --- a/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_no_params.groovy +++ /dev/null @@ -1 +0,0 @@ -return _subset_freq + _subset_size + _superset_freq + _superset_size diff --git a/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_with_params.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_with_params.groovy deleted file mode 100644 index 0099a531fd2..00000000000 --- a/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_with_params.groovy +++ /dev/null @@ -1 +0,0 @@ -return param*(_subset_freq + _subset_size + _superset_freq + _superset_size)/param diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/TemplateQueryBuilderTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/TemplateQueryBuilderTests.java index 96d6ce28dc1..dd5c0a18328 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/TemplateQueryBuilderTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/TemplateQueryBuilderTests.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.memory.MemoryIndex; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; @@ -33,16 +34,19 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.test.AbstractQueryTestCase; import org.junit.Before; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.function.Function; public class TemplateQueryBuilderTests extends AbstractQueryTestCase<TemplateQueryBuilder> { @@ -53,7 +57,39 @@ public class TemplateQueryBuilderTests extends AbstractQueryTestCase<TemplateQue @Override protected Collection<Class<? extends Plugin>> getPlugins() { - return Collections.singleton(MustachePlugin.class); + return Arrays.asList(MustachePlugin.class, CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends MockScriptPlugin { + + @Override + @SuppressWarnings("unchecked") + protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() { + Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>(); + + scripts.put("{ \"match_all\" : {}}", + s -> new BytesArray("{ \"match_all\" : {}}")); + + scripts.put("{ \"match_all\" : {\"_name\" : \"foobar\"}}", + s -> new BytesArray("{ \"match_all\" : {\"_name\" : \"foobar\"}}")); + + scripts.put("{\n" + + " \"term\" : {\n" + + " \"foo\" : {\n" + + " \"value\" : \"bar\",\n" + + " \"boost\" : 2.0\n" + + " }\n" + + " }\n" + + "}", s -> new BytesArray("{\n" + + " \"term\" : {\n" + + " \"foo\" : {\n" + + " \"value\" : \"bar\",\n" + + " \"boost\" : 2.0\n" + + " }\n" + + " }\n" + + "}")); + return scripts; + } } @Before @@ -68,7 +104,7 @@ public class TemplateQueryBuilderTests extends AbstractQueryTestCase<TemplateQue @Override protected TemplateQueryBuilder doCreateTestQueryBuilder() { - return new TemplateQueryBuilder(new Script(templateBase.toString(), ScriptType.INLINE, "mockscript", null, null)); + return new TemplateQueryBuilder(new Script(templateBase.toString(), ScriptType.INLINE, "mustache", null, null)); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java index b6fddaa427c..e036676677f 100644 --- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java +++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java @@ -34,6 +34,15 @@ import java.util.function.Function; /** * A mocked script engine that can be used for testing purpose. + * + * This script engine allows to define a set of predefined scripts that basically a combination of a key and a + * function: + * + * The key can be anything as long as it is a {@link String} and is used to resolve the scripts + * at compilation time. For inline scripts, the key can be a description of the script. For stored and file scripts, + * the source must match a key in the predefined set of scripts. + * + * The function is used to provide the result of the script execution and can return anything. */ public class MockScriptEngine implements ScriptEngineService { @@ -63,7 +72,13 @@ public class MockScriptEngine implements ScriptEngineService { @Override public Object compile(String name, String source, Map<String, String> params) { + // Scripts are always resolved using the script's source. For inline scripts, it's easy because they don't have names and the + // source is always provided. For stored and file scripts, the source of the script must match the key of a predefined script. Function<Map<String, Object>, Object> script = scripts.get(source); + if (script == null) { + throw new IllegalArgumentException("No pre defined script matching [" + source + "] for script with name [" + name + "], " + + "did you declare the mocked script?"); + } return new MockCompiledScript(name, params, source, script); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java index d864dc732c0..7dc358693fa 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java @@ -21,7 +21,6 @@ package org.elasticsearch.test; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.io.JsonStringEncoder; - import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; @@ -94,6 +93,7 @@ import org.elasticsearch.node.internal.InternalSettingsPreparer; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; @@ -861,7 +861,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>> new Class[]{Client.class}, clientInvocationHandler); NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); - ScriptModule scriptModule = newTestScriptModule(); + ScriptModule scriptModule = createScriptModule(pluginsService.filterPlugins(ScriptPlugin.class)); List<Setting<?>> scriptSettings = scriptModule.getSettings(); scriptSettings.addAll(pluginsService.getPluginSettings()); scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED); @@ -970,6 +970,20 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>> scriptService, indicesQueriesRegistry, client, null, state); } + ScriptModule createScriptModule(List<ScriptPlugin> scriptPlugins) { + if (scriptPlugins == null || scriptPlugins.isEmpty()) { + return newTestScriptModule(); + } + + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) + // no file watching, so we don't need a ResourceWatcherService + .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false) + .build(); + Environment environment = new Environment(settings); + return ScriptModule.create(settings, environment, null, scriptPlugins); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index c9ecf2ec639..95fd4186ed0 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -835,7 +835,8 @@ public abstract class ESTestCase extends LuceneTestCase { .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false) .build(); Environment environment = new Environment(settings); - return new ScriptModule(settings, environment, null, singletonList(new MockScriptEngine()), emptyList()); + MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap("1", script -> "1")); + return new ScriptModule(settings, environment, null, singletonList(scriptEngine), emptyList()); } /** Creates an IndicesModule for testing with the given mappers and metadata mappers. */