diff --git a/plugins/lang-python/README.md b/plugins/lang-python/README.md new file mode 100644 index 00000000000..6e6116b1028 --- /dev/null +++ b/plugins/lang-python/README.md @@ -0,0 +1,178 @@ +Python lang Plugin for Elasticsearch +================================== + +The Python (jython) language plugin allows to have `python` as the language of scripts to execute. + +In order to install the plugin, simply run: + +```sh +bin/plugin install elasticsearch/elasticsearch-lang-python/2.5.0 +``` + +You need to install a version matching your Elasticsearch version: + +| elasticsearch | Python Lang Plugin | Docs | +|---------------|-----------------------|------------| +| master | Build from source | See below | +| es-1.x | Build from source | [2.6.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.x/#version-260-snapshot-for-elasticsearch-1x) | +| es-1.5 | 2.5.0 | [2.5.0](https://github.com/elastic/elasticsearch-lang-python/tree/v2.5.0/#version-250-for-elasticsearch-15) | +| es-1.4 | 2.4.1 | [2.4.1](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.4.1/#version-241-for-elasticsearch-14) | +| es-1.3 | 2.3.1 | [2.3.1](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.1/#version-231-for-elasticsearch-13) | +| < 1.3.5 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.0/#version-230-for-elasticsearch-13) | +| es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.2.0/#python-lang-plugin-for-elasticsearch) | +| es-1.0 | 2.0.0 | [2.0.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.0.0/#python-lang-plugin-for-elasticsearch) | +| es-0.90 | 1.0.0 | [1.0.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v1.0.0/#python-lang-plugin-for-elasticsearch) | + +To build a `SNAPSHOT` version, you need to build it with Maven: + +```bash +mvn clean install +plugin --install lang-python \ + --url file:target/releases/elasticsearch-lang-python-X.X.X-SNAPSHOT.zip +``` + +User Guide +---------- + +Using python with function_score +-------------------------------- + +Let's say you want to use `function_score` API using `python`. Here is +a way of doing it: + +```sh +curl -XDELETE "http://localhost:9200/test" + +curl -XPUT "http://localhost:9200/test/doc/1" -d '{ + "num": 1.0 +}' + +curl -XPUT "http://localhost:9200/test/doc/2?refresh" -d '{ + "num": 2.0 +}' + +curl -XGET "http://localhost:9200/test/_search?pretty" -d' +{ + "query": { + "function_score": { + "script_score": { + "script": "doc[\"num\"].value * _score", + "lang": "python" + } + } + } +}' +``` + +gives + +```javascript +{ + // ... + "hits": { + "total": 2, + "max_score": 2, + "hits": [ + { + // ... + "_score": 2 + }, + { + // ... + "_score": 1 + } + ] + } +} +``` + +Using python with script_fields +------------------------------- + +```sh +curl -XDELETE "http://localhost:9200/test" + +curl -XPUT "http://localhost:9200/test/doc/1?refresh" -d' +{ + "obj1": { + "test": "something" + }, + "obj2": { + "arr2": [ "arr_value1", "arr_value2" ] + } +}' + +curl -XGET "http://localhost:9200/test/_search" -d' +{ + "script_fields": { + "s_obj1": { + "script": "_source[\"obj1\"]", "lang": "python" + }, + "s_obj1_test": { + "script": "_source[\"obj1\"][\"test\"]", "lang": "python" + }, + "s_obj2": { + "script": "_source[\"obj2\"]", "lang": "python" + }, + "s_obj2_arr2": { + "script": "_source[\"obj2\"][\"arr2\"]", "lang": "python" + } + } +}' +``` + +gives + +```javascript +{ + // ... + "hits": [ + { + // ... + "fields": { + "s_obj2_arr2": [ + [ + "arr_value1", + "arr_value2" + ] + ], + "s_obj1_test": [ + "something" + ], + "s_obj2": [ + { + "arr2": [ + "arr_value1", + "arr_value2" + ] + } + ], + "s_obj1": [ + { + "test": "something" + } + ] + } + } + ] +} +``` + +License +------- + + This software is licensed under the Apache 2 license, quoted below. + + Copyright 2009-2014 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); you may not + use this file except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations under + the License. diff --git a/plugins/lang-python/pom.xml b/plugins/lang-python/pom.xml new file mode 100644 index 00000000000..1d52498628c --- /dev/null +++ b/plugins/lang-python/pom.xml @@ -0,0 +1,42 @@ + + + 4.0.0 + + org.elasticsearch.plugin + elasticsearch-lang-python + + jar + Elasticsearch Python language plugin + The Python language plugin allows to have python as the language of scripts to execute. + + + org.elasticsearch + elasticsearch-plugin + 2.0.0-SNAPSHOT + + + + + + + + + + org.python + jython-standalone + 2.7.0 + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + + + diff --git a/plugins/lang-python/src/main/assemblies/plugin.xml b/plugins/lang-python/src/main/assemblies/plugin.xml new file mode 100644 index 00000000000..037ea9f7ee8 --- /dev/null +++ b/plugins/lang-python/src/main/assemblies/plugin.xml @@ -0,0 +1,26 @@ + + + plugin + + zip + + false + + + / + true + true + + org.elasticsearch:elasticsearch + + + + / + true + true + + org.python:jython-standalone + + + + \ No newline at end of file diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java b/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java new file mode 100644 index 00000000000..78f05311a6f --- /dev/null +++ b/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java @@ -0,0 +1,44 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugin.python; + +import org.elasticsearch.plugins.AbstractPlugin; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.python.PythonScriptEngineService; + +/** + * + */ +public class PythonPlugin extends AbstractPlugin { + + @Override + public String name() { + return "lang-python"; + } + + @Override + public String description() { + return "Adds support for writing scripts in Python"; + } + + public void onModule(ScriptModule module) { + module.addScriptEngine(PythonScriptEngineService.class); + } +} diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java new file mode 100644 index 00000000000..6138453925e --- /dev/null +++ b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java @@ -0,0 +1,242 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.python; + +import java.io.IOException; +import java.util.Map; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Scorer; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.LeafSearchScript; +import org.elasticsearch.script.ScoreAccessor; +import org.elasticsearch.script.ScriptEngineService; +import org.elasticsearch.script.SearchScript; +import org.elasticsearch.search.lookup.LeafSearchLookup; +import org.elasticsearch.search.lookup.SearchLookup; +import org.python.core.Py; +import org.python.core.PyCode; +import org.python.core.PyObject; +import org.python.core.PyStringMap; +import org.python.util.PythonInterpreter; + +/** + * + */ +//TODO we can optimize the case for Map similar to PyStringMap +public class PythonScriptEngineService extends AbstractComponent implements ScriptEngineService { + + private final PythonInterpreter interp; + + @Inject + public PythonScriptEngineService(Settings settings) { + super(settings); + + this.interp = PythonInterpreter.threadLocalStateInterpreter(null); + } + + @Override + public String[] types() { + return new String[]{"python", "py"}; + } + + @Override + public String[] extensions() { + return new String[]{"py"}; + } + + @Override + public boolean sandboxed() { + return false; + } + + @Override + public Object compile(String script) { + return interp.compile(script); + } + + @Override + public ExecutableScript executable(Object compiledScript, Map vars) { + return new PythonExecutableScript((PyCode) compiledScript, vars); + } + + @Override + public SearchScript search(final Object compiledScript, final SearchLookup lookup, @Nullable final Map vars) { + return new SearchScript() { + @Override + public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException { + final LeafSearchLookup leafLookup = lookup.getLeafSearchLookup(context); + return new PythonSearchScript((PyCode) compiledScript, vars, leafLookup); + } + }; + } + + @Override + public Object execute(Object compiledScript, Map vars) { + PyObject pyVars = Py.java2py(vars); + interp.setLocals(pyVars); + PyObject ret = interp.eval((PyCode) compiledScript); + if (ret == null) { + return null; + } + return ret.__tojava__(Object.class); + } + + @Override + public Object unwrap(Object value) { + return unwrapValue(value); + } + + @Override + public void close() { + interp.cleanup(); + } + + @Override + public void scriptRemoved(@Nullable CompiledScript compiledScript) { + // Nothing to do + } + + public class PythonExecutableScript implements ExecutableScript { + + private final PyCode code; + + private final PyStringMap pyVars; + + public PythonExecutableScript(PyCode code, Map vars) { + this.code = code; + this.pyVars = new PyStringMap(); + if (vars != null) { + for (Map.Entry entry : vars.entrySet()) { + pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue())); + } + } + } + + @Override + public void setNextVar(String name, Object value) { + pyVars.__setitem__(name, Py.java2py(value)); + } + + @Override + public Object run() { + interp.setLocals(pyVars); + PyObject ret = interp.eval(code); + if (ret == null) { + return null; + } + return ret.__tojava__(Object.class); + } + + @Override + public Object unwrap(Object value) { + return unwrapValue(value); + } + } + + public class PythonSearchScript implements LeafSearchScript { + + private final PyCode code; + + private final PyStringMap pyVars; + + private final LeafSearchLookup lookup; + + public PythonSearchScript(PyCode code, Map vars, LeafSearchLookup lookup) { + this.code = code; + this.pyVars = new PyStringMap(); + for (Map.Entry entry : lookup.asMap().entrySet()) { + pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue())); + } + if (vars != null) { + for (Map.Entry entry : vars.entrySet()) { + pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue())); + } + } + this.lookup = lookup; + } + + @Override + public void setScorer(Scorer scorer) { + pyVars.__setitem__("_score", Py.java2py(new ScoreAccessor(scorer))); + } + + @Override + public void setDocument(int doc) { + lookup.setDocument(doc); + } + + @Override + public void setSource(Map source) { + lookup.source().setSource(source); + } + + @Override + public void setNextVar(String name, Object value) { + pyVars.__setitem__(name, Py.java2py(value)); + } + + @Override + public Object run() { + interp.setLocals(pyVars); + PyObject ret = interp.eval(code); + if (ret == null) { + return null; + } + return ret.__tojava__(Object.class); + } + + @Override + public float runAsFloat() { + return ((Number) run()).floatValue(); + } + + @Override + public long runAsLong() { + return ((Number) run()).longValue(); + } + + @Override + public double runAsDouble() { + return ((Number) run()).doubleValue(); + } + + @Override + public Object unwrap(Object value) { + return unwrapValue(value); + } + } + + + public static Object unwrapValue(Object value) { + if (value == null) { + return null; + } else if (value instanceof PyObject) { + // seems like this is enough, inner PyDictionary will do the conversion for us for example, so expose it directly + return ((PyObject) value).__tojava__(Object.class); + } + return value; + } +} diff --git a/plugins/lang-python/src/main/resources/es-plugin.properties b/plugins/lang-python/src/main/resources/es-plugin.properties new file mode 100644 index 00000000000..b0ed0d5e3e0 --- /dev/null +++ b/plugins/lang-python/src/main/resources/es-plugin.properties @@ -0,0 +1,2 @@ +plugin=org.elasticsearch.plugin.python.PythonPlugin +version=${project.version} diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java new file mode 100644 index 00000000000..1621d22ac01 --- /dev/null +++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java @@ -0,0 +1,153 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.python; + +import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.test.ElasticsearchTestCase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +/** + * + */ +public class PythonScriptEngineTests extends ElasticsearchTestCase { + + private PythonScriptEngineService se; + + @Before + public void setup() { + se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); + } + + @After + public void close() { + // We need to clear some system properties + System.clearProperty("python.cachedir.skip"); + System.clearProperty("python.console.encoding"); + se.close(); + } + + @Test + public void testSimpleEquation() { + Map vars = new HashMap(); + Object o = se.execute(se.compile("1 + 2"), vars); + assertThat(((Number) o).intValue(), equalTo(3)); + } + + @Test + public void testMapAccess() { + Map vars = new HashMap(); + + Map obj2 = MapBuilder.newMapBuilder().put("prop2", "value2").map(); + Map obj1 = MapBuilder.newMapBuilder().put("prop1", "value1").put("obj2", obj2).put("l", Arrays.asList("2", "1")).map(); + vars.put("obj1", obj1); + Object o = se.execute(se.compile("obj1"), vars); + assertThat(o, instanceOf(Map.class)); + obj1 = (Map) o; + assertThat((String) obj1.get("prop1"), equalTo("value1")); + assertThat((String) ((Map) obj1.get("obj2")).get("prop2"), equalTo("value2")); + + o = se.execute(se.compile("obj1['l'][0]"), vars); + assertThat(((String) o), equalTo("2")); + } + + @Test + public void testObjectMapInter() { + Map vars = new HashMap(); + Map ctx = new HashMap(); + Map obj1 = new HashMap(); + obj1.put("prop1", "value1"); + ctx.put("obj1", obj1); + vars.put("ctx", ctx); + + se.execute(se.compile("ctx['obj2'] = { 'prop2' : 'value2' }; ctx['obj1']['prop1'] = 'uvalue1'"), vars); + ctx = (Map) se.unwrap(vars.get("ctx")); + assertThat(ctx.containsKey("obj1"), equalTo(true)); + assertThat((String) ((Map) ctx.get("obj1")).get("prop1"), equalTo("uvalue1")); + assertThat(ctx.containsKey("obj2"), equalTo(true)); + assertThat((String) ((Map) ctx.get("obj2")).get("prop2"), equalTo("value2")); + } + + @Test + public void testAccessListInScript() { + + Map vars = new HashMap(); + Map obj2 = MapBuilder.newMapBuilder().put("prop2", "value2").map(); + Map obj1 = MapBuilder.newMapBuilder().put("prop1", "value1").put("obj2", obj2).map(); + vars.put("l", Arrays.asList("1", "2", "3", obj1)); + +// Object o = se.execute(se.compile("l.length"), vars); +// assertThat(((Number) o).intValue(), equalTo(4)); + + Object o = se.execute(se.compile("l[0]"), vars); + assertThat(((String) o), equalTo("1")); + + o = se.execute(se.compile("l[3]"), vars); + obj1 = (Map) o; + assertThat((String) obj1.get("prop1"), equalTo("value1")); + assertThat((String) ((Map) obj1.get("obj2")).get("prop2"), equalTo("value2")); + + o = se.execute(se.compile("l[3]['prop1']"), vars); + assertThat(((String) o), equalTo("value1")); + } + + @Test + public void testChangingVarsCrossExecution1() { + Map vars = new HashMap(); + Map ctx = new HashMap(); + vars.put("ctx", ctx); + Object compiledScript = se.compile("ctx['value']"); + + ExecutableScript script = se.executable(compiledScript, vars); + ctx.put("value", 1); + Object o = script.run(); + assertThat(((Number) o).intValue(), equalTo(1)); + + ctx.put("value", 2); + o = script.run(); + assertThat(((Number) o).intValue(), equalTo(2)); + } + + @Test + public void testChangingVarsCrossExecution2() { + Map vars = new HashMap(); + Map ctx = new HashMap(); + Object compiledScript = se.compile("value"); + + ExecutableScript script = se.executable(compiledScript, vars); + script.setNextVar("value", 1); + Object o = script.run(); + assertThat(((Number) o).intValue(), equalTo(1)); + + script.setNextVar("value", 2); + o = script.run(); + assertThat(((Number) o).intValue(), equalTo(2)); + } +} diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java new file mode 100644 index 00000000000..9d53507388b --- /dev/null +++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.python; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.test.ElasticsearchTestCase; +import org.junit.After; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.hamcrest.Matchers.equalTo; + +/** + * + */ +public class PythonScriptMultiThreadedTest extends ElasticsearchTestCase { + + @After + public void close() { + // We need to clear some system properties + System.clearProperty("python.cachedir.skip"); + System.clearProperty("python.console.encoding"); + } + + @Test + public void testExecutableNoRuntimeParams() throws Exception { + final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); + final Object compiled = se.compile("x + y"); + final AtomicBoolean failed = new AtomicBoolean(); + + Thread[] threads = new Thread[4]; + final CountDownLatch latch = new CountDownLatch(threads.length); + final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1); + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(new Runnable() { + @Override + public void run() { + try { + barrier.await(); + long x = ThreadLocalRandom.current().nextInt(); + long y = ThreadLocalRandom.current().nextInt(); + long addition = x + y; + Map vars = new HashMap(); + vars.put("x", x); + vars.put("y", y); + ExecutableScript script = se.executable(compiled, vars); + for (int i = 0; i < 10000; i++) { + long result = ((Number) script.run()).longValue(); + assertThat(result, equalTo(addition)); + } + } catch (Throwable t) { + failed.set(true); + logger.error("failed", t); + } finally { + latch.countDown(); + } + } + }); + } + for (int i = 0; i < threads.length; i++) { + threads[i].start(); + } + barrier.await(); + latch.await(); + assertThat(failed.get(), equalTo(false)); + } + + +// @Test public void testExecutableWithRuntimeParams() throws Exception { +// final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); +// final Object compiled = se.compile("x + y"); +// final AtomicBoolean failed = new AtomicBoolean(); +// +// Thread[] threads = new Thread[50]; +// final CountDownLatch latch = new CountDownLatch(threads.length); +// final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1); +// for (int i = 0; i < threads.length; i++) { +// threads[i] = new Thread(new Runnable() { +// @Override public void run() { +// try { +// barrier.await(); +// long x = ThreadLocalRandom.current().nextInt(); +// Map vars = new HashMap(); +// vars.put("x", x); +// ExecutableScript script = se.executable(compiled, vars); +// Map runtimeVars = new HashMap(); +// for (int i = 0; i < 100000; i++) { +// long y = ThreadLocalRandom.current().nextInt(); +// long addition = x + y; +// runtimeVars.put("y", y); +// long result = ((Number) script.run(runtimeVars)).longValue(); +// assertThat(result, equalTo(addition)); +// } +// } catch (Throwable t) { +// failed.set(true); +// logger.error("failed", t); +// } finally { +// latch.countDown(); +// } +// } +// }); +// } +// for (int i = 0; i < threads.length; i++) { +// threads[i].start(); +// } +// barrier.await(); +// latch.await(); +// assertThat(failed.get(), equalTo(false)); +// } + + @Test + public void testExecute() throws Exception { + final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); + final Object compiled = se.compile("x + y"); + final AtomicBoolean failed = new AtomicBoolean(); + + Thread[] threads = new Thread[4]; + final CountDownLatch latch = new CountDownLatch(threads.length); + final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1); + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(new Runnable() { + @Override + public void run() { + try { + barrier.await(); + Map runtimeVars = new HashMap(); + for (int i = 0; i < 10000; i++) { + long x = ThreadLocalRandom.current().nextInt(); + long y = ThreadLocalRandom.current().nextInt(); + long addition = x + y; + runtimeVars.put("x", x); + runtimeVars.put("y", y); + long result = ((Number) se.execute(compiled, runtimeVars)).longValue(); + assertThat(result, equalTo(addition)); + } + } catch (Throwable t) { + failed.set(true); + logger.error("failed", t); + } finally { + latch.countDown(); + } + } + }); + } + for (int i = 0; i < threads.length; i++) { + threads[i].start(); + } + barrier.await(); + latch.await(); + assertThat(failed.get(), equalTo(false)); + } +} diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java new file mode 100644 index 00000000000..0ff68beea3e --- /dev/null +++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java @@ -0,0 +1,311 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.python; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.hamcrest.CoreMatchers; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.client.Requests.searchRequest; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.QueryBuilders.*; +import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; +import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.equalTo; + +/** + * + */ +@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.SUITE) +public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .build(); + } + + @After + public void close() { + // We need to clear some system properties + System.clearProperty("python.cachedir.skip"); + System.clearProperty("python.console.encoding"); + } + + @Test + public void testPythonFilter() throws Exception { + createIndex("test"); + index("test", "type1", "1", jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject()); + flush(); + index("test", "type1", "2", jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject()); + flush(); + index("test", "type1", "3", jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).endObject()); + refresh(); + + logger.info(" --> running doc['num1'].value > 1"); + SearchResponse response = client().prepareSearch() + .setQuery(filteredQuery(matchAllQuery(), scriptQuery("doc['num1'].value > 1").lang("python"))) + .addSort("num1", SortOrder.ASC) + .addScriptField("sNum1", "python", "doc['num1'].value", null) + .execute().actionGet(); + + assertThat(response.getHits().totalHits(), equalTo(2l)); + assertThat(response.getHits().getAt(0).id(), equalTo("2")); + assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0)); + assertThat(response.getHits().getAt(1).id(), equalTo("3")); + assertThat((Double) response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(3.0)); + + logger.info(" --> running doc['num1'].value > param1"); + response = client().prepareSearch() + .setQuery(filteredQuery(matchAllQuery(), scriptQuery("doc['num1'].value > param1").lang("python").addParam("param1", 2))) + .addSort("num1", SortOrder.ASC) + .addScriptField("sNum1", "python", "doc['num1'].value", null) + .execute().actionGet(); + + assertThat(response.getHits().totalHits(), equalTo(1l)); + assertThat(response.getHits().getAt(0).id(), equalTo("3")); + assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(3.0)); + + logger.info(" --> running doc['num1'].value > param1"); + response = client().prepareSearch() + .setQuery(filteredQuery(matchAllQuery(), scriptQuery("doc['num1'].value > param1").lang("python").addParam("param1", -1))) + .addSort("num1", SortOrder.ASC) + .addScriptField("sNum1", "python", "doc['num1'].value", null) + .execute().actionGet(); + + assertThat(response.getHits().totalHits(), equalTo(3l)); + assertThat(response.getHits().getAt(0).id(), equalTo("1")); + assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0)); + assertThat(response.getHits().getAt(1).id(), equalTo("2")); + assertThat((Double) response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0)); + assertThat(response.getHits().getAt(2).id(), equalTo("3")); + assertThat((Double) response.getHits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0)); + } + + @Test + public void testScriptFieldUsingSource() throws Exception { + createIndex("test"); + index("test", "type1", "1", + jsonBuilder().startObject() + .startObject("obj1").field("test", "something").endObject() + .startObject("obj2").startArray("arr2").value("arr_value1").value("arr_value2").endArray().endObject() + .endObject()); + refresh(); + + SearchResponse response = client().prepareSearch() + .setQuery(matchAllQuery()) + .addScriptField("s_obj1", "python", "_source['obj1']", null) + .addScriptField("s_obj1_test", "python", "_source['obj1']['test']", null) + .addScriptField("s_obj2", "python", "_source['obj2']", null) + .addScriptField("s_obj2_arr2", "python", "_source['obj2']['arr2']", null) + .execute().actionGet(); + + Map sObj1 = (Map) response.getHits().getAt(0).field("s_obj1").value(); + assertThat(sObj1.get("test").toString(), equalTo("something")); + assertThat(response.getHits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something")); + + Map sObj2 = (Map) response.getHits().getAt(0).field("s_obj2").value(); + List sObj2Arr2 = (List) sObj2.get("arr2"); + assertThat(sObj2Arr2.size(), equalTo(2)); + assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1")); + assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2")); + + sObj2Arr2 = (List) response.getHits().getAt(0).field("s_obj2_arr2").values(); + assertThat(sObj2Arr2.size(), equalTo(2)); + assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1")); + assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2")); + } + + @Test + public void testCustomScriptBoost() throws Exception { + createIndex("test"); + index("test", "type1", "1", jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject()); + index("test", "type1", "2", jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject()); + refresh(); + + logger.info("--- QUERY_THEN_FETCH"); + + logger.info(" --> running doc['num1'].value"); + SearchResponse response = client().search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("doc['num1'].value").lang("python")))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); + + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + assertThat(response.getHits().getAt(0).id(), equalTo("2")); + assertThat(response.getHits().getAt(1).id(), equalTo("1")); + + logger.info(" --> running -doc['num1'].value"); + response = client().search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("-doc['num1'].value").lang("python")))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); + + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + assertThat(response.getHits().getAt(0).id(), equalTo("1")); + assertThat(response.getHits().getAt(1).id(), equalTo("2")); + + + logger.info(" --> running doc['num1'].value * _score"); + response = client().search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("doc['num1'].value * _score.doubleValue()").lang("python")))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); + + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + assertThat(response.getHits().getAt(0).id(), equalTo("2")); + assertThat(response.getHits().getAt(1).id(), equalTo("1")); + + logger.info(" --> running param1 * param2 * _score"); + response = client().search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("param1 * param2 * _score.doubleValue()").param("param1", 2).param("param2", 2).lang("python")))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); + + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + } + + /** + * Test case for #4: https://github.com/elasticsearch/elasticsearch-lang-python/issues/4 + * Update request that uses python script with no parameters fails with NullPointerException + * @throws Exception + */ + @Test + public void testPythonEmptyParameters() throws Exception { + createIndex("test"); + index("test", "type1", "1", jsonBuilder().startObject().field("myfield", "foo").endObject()); + refresh(); + + client().prepareUpdate("test", "type1", "1").setScriptLang("python") + .setScript("ctx[\"_source\"][\"myfield\"]=\"bar\"", ScriptService.ScriptType.INLINE) + .execute().actionGet(); + refresh(); + + Object value = get("test", "type1", "1").getSourceAsMap().get("myfield"); + assertThat(value instanceof String, is(true)); + + assertThat((String) value, CoreMatchers.equalTo("bar")); + } + + @Test + public void testScriptScoresNested() throws IOException { + createIndex("index"); + ensureYellow(); + index("index", "testtype", "1", jsonBuilder().startObject().field("dummy_field", 1).endObject()); + refresh(); + SearchResponse response = client().search( + searchRequest().source( + searchSource().query( + functionScoreQuery( + functionScoreQuery( + functionScoreQuery().add(scriptFunction("1").lang("python"))) + .add(scriptFunction("_score.doubleValue()").lang("python"))) + .add(scriptFunction("_score.doubleValue()").lang("python") + ) + ) + ) + ).actionGet(); + assertSearchResponse(response); + assertThat(response.getHits().getAt(0).score(), equalTo(1.0f)); + } + + @Test + public void testScriptScoresWithAgg() throws IOException { + createIndex("index"); + ensureYellow(); + index("index", "testtype", "1", jsonBuilder().startObject().field("dummy_field", 1).endObject()); + refresh(); + SearchResponse response = client().search( + searchRequest().source( + searchSource().query( + functionScoreQuery() + .add(scriptFunction("_score.doubleValue()").lang("python") + ) + ).aggregation(terms("score_agg").script("_score.doubleValue()").lang("python")) + ) + ).actionGet(); + assertSearchResponse(response); + assertThat(response.getHits().getAt(0).score(), equalTo(1.0f)); + assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getKeyAsNumber().floatValue(), Matchers.is(1f)); + assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getDocCount(), Matchers.is(1l)); + } + + /** + * Test case for #19: https://github.com/elasticsearch/elasticsearch-lang-python/issues/19 + * Multi-line or multi-statement Python scripts raise NullPointerException + */ + @Test + public void testPythonMultiLines() throws Exception { + createIndex("test"); + index("test", "type1", "1", jsonBuilder().startObject().field("myfield", "foo").endObject()); + refresh(); + + client().prepareUpdate("test", "type1", "1").setScriptLang("python") + .setScript("a=42; ctx[\"_source\"][\"myfield\"]=\"bar\"", ScriptService.ScriptType.INLINE) + .execute().actionGet(); + refresh(); + + Object value = get("test", "type1", "1").getSourceAsMap().get("myfield"); + assertThat(value instanceof String, is(true)); + + assertThat((String) value, CoreMatchers.equalTo("bar")); + } + +} diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/SimpleBench.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/SimpleBench.java new file mode 100644 index 00000000000..583bab163fa --- /dev/null +++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/SimpleBench.java @@ -0,0 +1,71 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.python; + +import org.elasticsearch.common.StopWatch; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ExecutableScript; + +import java.util.HashMap; +import java.util.Map; + +/** + * + */ +public class SimpleBench { + + public static void main(String[] args) { + PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); + Object compiled = se.compile("x + y"); + + Map vars = new HashMap(); + // warm up + for (int i = 0; i < 1000; i++) { + vars.put("x", i); + vars.put("y", i + 1); + se.execute(compiled, vars); + } + + final long ITER = 100000; + + StopWatch stopWatch = new StopWatch().start(); + for (long i = 0; i < ITER; i++) { + se.execute(compiled, vars); + } + System.out.println("Execute Took: " + stopWatch.stop().lastTaskTime()); + + stopWatch = new StopWatch().start(); + ExecutableScript executableScript = se.executable(compiled, vars); + for (long i = 0; i < ITER; i++) { + executableScript.run(); + } + System.out.println("Executable Took: " + stopWatch.stop().lastTaskTime()); + + stopWatch = new StopWatch().start(); + executableScript = se.executable(compiled, vars); + for (long i = 0; i < ITER; i++) { + for (Map.Entry entry : vars.entrySet()) { + executableScript.setNextVar(entry.getKey(), entry.getValue()); + } + executableScript.run(); + } + System.out.println("Executable (vars) Took: " + stopWatch.stop().lastTaskTime()); + } +}