honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

migrate branch for lang-python

This commit is contained in:
Simon Willnauer 2015-06-09 15:17:23 +02:00
parent ea07e5ba94 a1b410652d
commit a04a9777d4
10 changed files with 1245 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

178
plugins/lang-python/README.md Normal file
View File

@ -0,0 +1,178 @@
Python lang Plugin for Elasticsearch
==================================
The Python (jython) language plugin allows to have `python` as the language of scripts to execute.
In order to install the plugin, simply run:
```sh
bin/plugin install elasticsearch/elasticsearch-lang-python/2.5.0
```
You need to install a version matching your Elasticsearch version:
| elasticsearch | Python Lang Plugin | Docs |
|---------------|-----------------------|------------|
| master | Build from source | See below |
| es-1.x | Build from source | [2.6.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.x/#version-260-snapshot-for-elasticsearch-1x) |
| es-1.5 | 2.5.0 | [2.5.0](https://github.com/elastic/elasticsearch-lang-python/tree/v2.5.0/#version-250-for-elasticsearch-15) |
| es-1.4 | 2.4.1 | [2.4.1](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.4.1/#version-241-for-elasticsearch-14) |
| es-1.3 | 2.3.1 | [2.3.1](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.1/#version-231-for-elasticsearch-13) |
| < 1.3.5 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.0/#version-230-for-elasticsearch-13) |
| es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.2.0/#python-lang-plugin-for-elasticsearch) |
| es-1.0 | 2.0.0 | [2.0.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.0.0/#python-lang-plugin-for-elasticsearch) |
| es-0.90 | 1.0.0 | [1.0.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v1.0.0/#python-lang-plugin-for-elasticsearch) |
To build a `SNAPSHOT` version, you need to build it with Maven:
```bash
mvn clean install
plugin --install lang-python \
--url file:target/releases/elasticsearch-lang-python-X.X.X-SNAPSHOT.zip
```
User Guide
----------
Using python with function_score
--------------------------------
Let's say you want to use `function_score` API using `python`. Here is
a way of doing it:
```sh
curl -XDELETE "http://localhost:9200/test"
curl -XPUT "http://localhost:9200/test/doc/1" -d '{
"num": 1.0
}'
curl -XPUT "http://localhost:9200/test/doc/2?refresh" -d '{
"num": 2.0
}'
curl -XGET "http://localhost:9200/test/_search?pretty" -d'
{
"query": {
"function_score": {
"script_score": {
"script": "doc[\"num\"].value * _score",
"lang": "python"
}
}
}
}'
```
gives
```javascript
{
// ...
"hits": {
"total": 2,
"max_score": 2,
"hits": [
{
// ...
"_score": 2
},
{
// ...
"_score": 1
}
]
}
}
```
Using python with script_fields
-------------------------------
```sh
curl -XDELETE "http://localhost:9200/test"
curl -XPUT "http://localhost:9200/test/doc/1?refresh" -d'
{
"obj1": {
"test": "something"
},
"obj2": {
"arr2": [ "arr_value1", "arr_value2" ]
}
}'
curl -XGET "http://localhost:9200/test/_search" -d'
{
"script_fields": {
"s_obj1": {
"script": "_source[\"obj1\"]", "lang": "python"
},
"s_obj1_test": {
"script": "_source[\"obj1\"][\"test\"]", "lang": "python"
},
"s_obj2": {
"script": "_source[\"obj2\"]", "lang": "python"
},
"s_obj2_arr2": {
"script": "_source[\"obj2\"][\"arr2\"]", "lang": "python"
}
}
}'
```
gives
```javascript
{
// ...
"hits": [
{
// ...
"fields": {
"s_obj2_arr2": [
[
"arr_value1",
"arr_value2"
]
],
"s_obj1_test": [
"something"
],
"s_obj2": [
{
"arr2": [
"arr_value1",
"arr_value2"
]
}
],
"s_obj1": [
{
"test": "something"
}
]
}
}
]
}
```
License
-------
This software is licensed under the Apache 2 license, quoted below.
Copyright 2009-2014 Elasticsearch <http://www.elasticsearch.org>
Licensed under the Apache License, Version 2.0 (the "License"); you may not
use this file except in compliance with the License. You may obtain a copy of
the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations under
the License.

42
plugins/lang-python/pom.xml Normal file
View File

@ -0,0 +1,42 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.elasticsearch.plugin</groupId>
<artifactId>elasticsearch-lang-python</artifactId>
<packaging>jar</packaging>
<name>Elasticsearch Python language plugin</name>
<description>The Python language plugin allows to have python as the language of scripts to execute.</description>
<parent>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch-plugin</artifactId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<properties>
<!-- You can add any specific project property here -->
</properties>
<dependencies>
<!-- Jython -->
<dependency>
<groupId>org.python</groupId>
<artifactId>jython-standalone</artifactId>
<version>2.7.0</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

26
plugins/lang-python/src/main/assemblies/plugin.xml Normal file
View File

@ -0,0 +1,26 @@
<?xml version="1.0"?>
<assembly>
<id>plugin</id>
<formats>
<format>zip</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<dependencySets>
<dependencySet>
<outputDirectory>/</outputDirectory>
<useProjectArtifact>true</useProjectArtifact>
<useTransitiveFiltering>true</useTransitiveFiltering>
<excludes>
<exclude>org.elasticsearch:elasticsearch</exclude>
</excludes>
</dependencySet>
<dependencySet>
<outputDirectory>/</outputDirectory>
<useProjectArtifact>true</useProjectArtifact>
<useTransitiveFiltering>true</useTransitiveFiltering>
<includes>
<include>org.python:jython-standalone</include>
</includes>
</dependencySet>
</dependencySets>
</assembly>

44
plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java Normal file
View File

@ -0,0 +1,44 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.plugin.python;
import org.elasticsearch.plugins.AbstractPlugin;
import org.elasticsearch.script.ScriptModule;
import org.elasticsearch.script.python.PythonScriptEngineService;
/**
*
*/
public class PythonPlugin extends AbstractPlugin {
@Override
public String name() {
return "lang-python";
}
@Override
public String description() {
return "Adds support for writing scripts in Python";
}
public void onModule(ScriptModule module) {
module.addScriptEngine(PythonScriptEngineService.class);
}
}

242
plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java Normal file
View File

@ -0,0 +1,242 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.python;
import java.io.IOException;
import java.util.Map;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.Scorer;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.script.LeafSearchScript;
import org.elasticsearch.script.ScoreAccessor;
import org.elasticsearch.script.ScriptEngineService;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.lookup.LeafSearchLookup;
import org.elasticsearch.search.lookup.SearchLookup;
import org.python.core.Py;
import org.python.core.PyCode;
import org.python.core.PyObject;
import org.python.core.PyStringMap;
import org.python.util.PythonInterpreter;
/**
*
*/
//TODO we can optimize the case for Map<String, Object> similar to PyStringMap
public class PythonScriptEngineService extends AbstractComponent implements ScriptEngineService {
private final PythonInterpreter interp;
@Inject
public PythonScriptEngineService(Settings settings) {
super(settings);
this.interp = PythonInterpreter.threadLocalStateInterpreter(null);
}
@Override
public String[] types() {
return new String[]{"python", "py"};
}
@Override
public String[] extensions() {
return new String[]{"py"};
}
@Override
public boolean sandboxed() {
return false;
}
@Override
public Object compile(String script) {
return interp.compile(script);
}
@Override
public ExecutableScript executable(Object compiledScript, Map<String, Object> vars) {
return new PythonExecutableScript((PyCode) compiledScript, vars);
}
@Override
public SearchScript search(final Object compiledScript, final SearchLookup lookup, @Nullable final Map<String, Object> vars) {
return new SearchScript() {
@Override
public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException {
final LeafSearchLookup leafLookup = lookup.getLeafSearchLookup(context);
return new PythonSearchScript((PyCode) compiledScript, vars, leafLookup);
}
};
}
@Override
public Object execute(Object compiledScript, Map<String, Object> vars) {
PyObject pyVars = Py.java2py(vars);
interp.setLocals(pyVars);
PyObject ret = interp.eval((PyCode) compiledScript);
if (ret == null) {
return null;
}
return ret.__tojava__(Object.class);
}
@Override
public Object unwrap(Object value) {
return unwrapValue(value);
}
@Override
public void close() {
interp.cleanup();
}
@Override
public void scriptRemoved(@Nullable CompiledScript compiledScript) {
// Nothing to do
}
public class PythonExecutableScript implements ExecutableScript {
private final PyCode code;
private final PyStringMap pyVars;
public PythonExecutableScript(PyCode code, Map<String, Object> vars) {
this.code = code;
this.pyVars = new PyStringMap();
if (vars != null) {
for (Map.Entry<String, Object> entry : vars.entrySet()) {
pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue()));
}
}
}
@Override
public void setNextVar(String name, Object value) {
pyVars.__setitem__(name, Py.java2py(value));
}
@Override
public Object run() {
interp.setLocals(pyVars);
PyObject ret = interp.eval(code);
if (ret == null) {
return null;
}
return ret.__tojava__(Object.class);
}
@Override
public Object unwrap(Object value) {
return unwrapValue(value);
}
}
public class PythonSearchScript implements LeafSearchScript {
private final PyCode code;
private final PyStringMap pyVars;
private final LeafSearchLookup lookup;
public PythonSearchScript(PyCode code, Map<String, Object> vars, LeafSearchLookup lookup) {
this.code = code;
this.pyVars = new PyStringMap();
for (Map.Entry<String, Object> entry : lookup.asMap().entrySet()) {
pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue()));
}
if (vars != null) {
for (Map.Entry<String, Object> entry : vars.entrySet()) {
pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue()));
}
}
this.lookup = lookup;
}
@Override
public void setScorer(Scorer scorer) {
pyVars.__setitem__("_score", Py.java2py(new ScoreAccessor(scorer)));
}
@Override
public void setDocument(int doc) {
lookup.setDocument(doc);
}
@Override
public void setSource(Map<String, Object> source) {
lookup.source().setSource(source);
}
@Override
public void setNextVar(String name, Object value) {
pyVars.__setitem__(name, Py.java2py(value));
}
@Override
public Object run() {
interp.setLocals(pyVars);
PyObject ret = interp.eval(code);
if (ret == null) {
return null;
}
return ret.__tojava__(Object.class);
}
@Override
public float runAsFloat() {
return ((Number) run()).floatValue();
}
@Override
public long runAsLong() {
return ((Number) run()).longValue();
}
@Override
public double runAsDouble() {
return ((Number) run()).doubleValue();
}
@Override
public Object unwrap(Object value) {
return unwrapValue(value);
}
}
public static Object unwrapValue(Object value) {
if (value == null) {
return null;
} else if (value instanceof PyObject) {
// seems like this is enough, inner PyDictionary will do the conversion for us for example, so expose it directly
return ((PyObject) value).__tojava__(Object.class);
}
return value;
}
}

2
plugins/lang-python/src/main/resources/es-plugin.properties Normal file
View File

@ -0,0 +1,2 @@
plugin=org.elasticsearch.plugin.python.PythonPlugin
version=${project.version}

153
plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java Normal file
View File

@ -0,0 +1,153 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.python;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
/**
*
*/
public class PythonScriptEngineTests extends ElasticsearchTestCase {
private PythonScriptEngineService se;
@Before
public void setup() {
se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
}
@After
public void close() {
// We need to clear some system properties
System.clearProperty("python.cachedir.skip");
System.clearProperty("python.console.encoding");
se.close();
}
@Test
public void testSimpleEquation() {
Map<String, Object> vars = new HashMap<String, Object>();
Object o = se.execute(se.compile("1 + 2"), vars);
assertThat(((Number) o).intValue(), equalTo(3));
}
@Test
public void testMapAccess() {
Map<String, Object> vars = new HashMap<String, Object>();
Map<String, Object> obj2 = MapBuilder.<String, Object>newMapBuilder().put("prop2", "value2").map();
Map<String, Object> obj1 = MapBuilder.<String, Object>newMapBuilder().put("prop1", "value1").put("obj2", obj2).put("l", Arrays.asList("2", "1")).map();
vars.put("obj1", obj1);
Object o = se.execute(se.compile("obj1"), vars);
assertThat(o, instanceOf(Map.class));
obj1 = (Map<String, Object>) o;
assertThat((String) obj1.get("prop1"), equalTo("value1"));
assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
o = se.execute(se.compile("obj1['l'][0]"), vars);
assertThat(((String) o), equalTo("2"));
}
@Test
public void testObjectMapInter() {
Map<String, Object> vars = new HashMap<String, Object>();
Map<String, Object> ctx = new HashMap<String, Object>();
Map<String, Object> obj1 = new HashMap<String, Object>();
obj1.put("prop1", "value1");
ctx.put("obj1", obj1);
vars.put("ctx", ctx);
se.execute(se.compile("ctx['obj2'] = { 'prop2' : 'value2' }; ctx['obj1']['prop1'] = 'uvalue1'"), vars);
ctx = (Map<String, Object>) se.unwrap(vars.get("ctx"));
assertThat(ctx.containsKey("obj1"), equalTo(true));
assertThat((String) ((Map<String, Object>) ctx.get("obj1")).get("prop1"), equalTo("uvalue1"));
assertThat(ctx.containsKey("obj2"), equalTo(true));
assertThat((String) ((Map<String, Object>) ctx.get("obj2")).get("prop2"), equalTo("value2"));
}
@Test
public void testAccessListInScript() {
Map<String, Object> vars = new HashMap<String, Object>();
Map<String, Object> obj2 = MapBuilder.<String, Object>newMapBuilder().put("prop2", "value2").map();
Map<String, Object> obj1 = MapBuilder.<String, Object>newMapBuilder().put("prop1", "value1").put("obj2", obj2).map();
vars.put("l", Arrays.asList("1", "2", "3", obj1));
// Object o = se.execute(se.compile("l.length"), vars);
// assertThat(((Number) o).intValue(), equalTo(4));
Object o = se.execute(se.compile("l[0]"), vars);
assertThat(((String) o), equalTo("1"));
o = se.execute(se.compile("l[3]"), vars);
obj1 = (Map<String, Object>) o;
assertThat((String) obj1.get("prop1"), equalTo("value1"));
assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
o = se.execute(se.compile("l[3]['prop1']"), vars);
assertThat(((String) o), equalTo("value1"));
}
@Test
public void testChangingVarsCrossExecution1() {
Map<String, Object> vars = new HashMap<String, Object>();
Map<String, Object> ctx = new HashMap<String, Object>();
vars.put("ctx", ctx);
Object compiledScript = se.compile("ctx['value']");
ExecutableScript script = se.executable(compiledScript, vars);
ctx.put("value", 1);
Object o = script.run();
assertThat(((Number) o).intValue(), equalTo(1));
ctx.put("value", 2);
o = script.run();
assertThat(((Number) o).intValue(), equalTo(2));
}
@Test
public void testChangingVarsCrossExecution2() {
Map<String, Object> vars = new HashMap<String, Object>();
Map<String, Object> ctx = new HashMap<String, Object>();
Object compiledScript = se.compile("value");
ExecutableScript script = se.executable(compiledScript, vars);
script.setNextVar("value", 1);
Object o = script.run();
assertThat(((Number) o).intValue(), equalTo(1));
script.setNextVar("value", 2);
o = script.run();
assertThat(((Number) o).intValue(), equalTo(2));
}
}

176
plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java Normal file
View File

@ -0,0 +1,176 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.python;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.junit.After;
import org.junit.Test;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.hamcrest.Matchers.equalTo;
/**
*
*/
public class PythonScriptMultiThreadedTest extends ElasticsearchTestCase {
@After
public void close() {
// We need to clear some system properties
System.clearProperty("python.cachedir.skip");
System.clearProperty("python.console.encoding");
}
@Test
public void testExecutableNoRuntimeParams() throws Exception {
final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
final Object compiled = se.compile("x + y");
final AtomicBoolean failed = new AtomicBoolean();
Thread[] threads = new Thread[4];
final CountDownLatch latch = new CountDownLatch(threads.length);
final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1);
for (int i = 0; i < threads.length; i++) {
threads[i] = new Thread(new Runnable() {
@Override
public void run() {
try {
barrier.await();
long x = ThreadLocalRandom.current().nextInt();
long y = ThreadLocalRandom.current().nextInt();
long addition = x + y;
Map<String, Object> vars = new HashMap<String, Object>();
vars.put("x", x);
vars.put("y", y);
ExecutableScript script = se.executable(compiled, vars);
for (int i = 0; i < 10000; i++) {
long result = ((Number) script.run()).longValue();
assertThat(result, equalTo(addition));
}
} catch (Throwable t) {
failed.set(true);
logger.error("failed", t);
} finally {
latch.countDown();
}
}
});
}
for (int i = 0; i < threads.length; i++) {
threads[i].start();
}
barrier.await();
latch.await();
assertThat(failed.get(), equalTo(false));
}
// @Test public void testExecutableWithRuntimeParams() throws Exception {
// final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
// final Object compiled = se.compile("x + y");
// final AtomicBoolean failed = new AtomicBoolean();
//
// Thread[] threads = new Thread[50];
// final CountDownLatch latch = new CountDownLatch(threads.length);
// final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1);
// for (int i = 0; i < threads.length; i++) {
// threads[i] = new Thread(new Runnable() {
// @Override public void run() {
// try {
// barrier.await();
// long x = ThreadLocalRandom.current().nextInt();
// Map<String, Object> vars = new HashMap<String, Object>();
// vars.put("x", x);
// ExecutableScript script = se.executable(compiled, vars);
// Map<String, Object> runtimeVars = new HashMap<String, Object>();
// for (int i = 0; i < 100000; i++) {
// long y = ThreadLocalRandom.current().nextInt();
// long addition = x + y;
// runtimeVars.put("y", y);
// long result = ((Number) script.run(runtimeVars)).longValue();
// assertThat(result, equalTo(addition));
// }
// } catch (Throwable t) {
// failed.set(true);
// logger.error("failed", t);
// } finally {
// latch.countDown();
// }
// }
// });
// }
// for (int i = 0; i < threads.length; i++) {
// threads[i].start();
// }
// barrier.await();
// latch.await();
// assertThat(failed.get(), equalTo(false));
// }
@Test
public void testExecute() throws Exception {
final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
final Object compiled = se.compile("x + y");
final AtomicBoolean failed = new AtomicBoolean();
Thread[] threads = new Thread[4];
final CountDownLatch latch = new CountDownLatch(threads.length);
final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1);
for (int i = 0; i < threads.length; i++) {
threads[i] = new Thread(new Runnable() {
@Override
public void run() {
try {
barrier.await();
Map<String, Object> runtimeVars = new HashMap<String, Object>();
for (int i = 0; i < 10000; i++) {
long x = ThreadLocalRandom.current().nextInt();
long y = ThreadLocalRandom.current().nextInt();
long addition = x + y;
runtimeVars.put("x", x);
runtimeVars.put("y", y);
long result = ((Number) se.execute(compiled, runtimeVars)).longValue();
assertThat(result, equalTo(addition));
}
} catch (Throwable t) {
failed.set(true);
logger.error("failed", t);
} finally {
latch.countDown();
}
}
});
}
for (int i = 0; i < threads.length; i++) {
threads[i].start();
}
barrier.await();
latch.await();
assertThat(failed.get(), equalTo(false));
}
}

311
plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java Normal file
View File

@ -0,0 +1,311 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.python;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.plugins.PluginsService;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.hamcrest.CoreMatchers;
import org.hamcrest.Matchers;
import org.junit.After;
import org.junit.Test;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.client.Requests.searchRequest;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.*;
import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction;
import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.Matchers.equalTo;
/**
*
*/
@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.SUITE)
public class PythonScriptSearchTests extends ElasticsearchIntegrationTest {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true)
.build();
}
@After
public void close() {
// We need to clear some system properties
System.clearProperty("python.cachedir.skip");
System.clearProperty("python.console.encoding");
}
@Test
public void testPythonFilter() throws Exception {
createIndex("test");
index("test", "type1", "1", jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject());
flush();
index("test", "type1", "2", jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject());
flush();
index("test", "type1", "3", jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).endObject());
refresh();
logger.info(" --> running doc['num1'].value > 1");
SearchResponse response = client().prepareSearch()
.setQuery(filteredQuery(matchAllQuery(), scriptQuery("doc['num1'].value > 1").lang("python")))
.addSort("num1", SortOrder.ASC)
.addScriptField("sNum1", "python", "doc['num1'].value", null)
.execute().actionGet();
assertThat(response.getHits().totalHits(), equalTo(2l));
assertThat(response.getHits().getAt(0).id(), equalTo("2"));
assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0));
assertThat(response.getHits().getAt(1).id(), equalTo("3"));
assertThat((Double) response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(3.0));
logger.info(" --> running doc['num1'].value > param1");
response = client().prepareSearch()
.setQuery(filteredQuery(matchAllQuery(), scriptQuery("doc['num1'].value > param1").lang("python").addParam("param1", 2)))
.addSort("num1", SortOrder.ASC)
.addScriptField("sNum1", "python", "doc['num1'].value", null)
.execute().actionGet();
assertThat(response.getHits().totalHits(), equalTo(1l));
assertThat(response.getHits().getAt(0).id(), equalTo("3"));
assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(3.0));
logger.info(" --> running doc['num1'].value > param1");
response = client().prepareSearch()
.setQuery(filteredQuery(matchAllQuery(), scriptQuery("doc['num1'].value > param1").lang("python").addParam("param1", -1)))
.addSort("num1", SortOrder.ASC)
.addScriptField("sNum1", "python", "doc['num1'].value", null)
.execute().actionGet();
assertThat(response.getHits().totalHits(), equalTo(3l));
assertThat(response.getHits().getAt(0).id(), equalTo("1"));
assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0));
assertThat(response.getHits().getAt(1).id(), equalTo("2"));
assertThat((Double) response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0));
assertThat(response.getHits().getAt(2).id(), equalTo("3"));
assertThat((Double) response.getHits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0));
}
@Test
public void testScriptFieldUsingSource() throws Exception {
createIndex("test");
index("test", "type1", "1",
jsonBuilder().startObject()
.startObject("obj1").field("test", "something").endObject()
.startObject("obj2").startArray("arr2").value("arr_value1").value("arr_value2").endArray().endObject()
.endObject());
refresh();
SearchResponse response = client().prepareSearch()
.setQuery(matchAllQuery())
.addScriptField("s_obj1", "python", "_source['obj1']", null)
.addScriptField("s_obj1_test", "python", "_source['obj1']['test']", null)
.addScriptField("s_obj2", "python", "_source['obj2']", null)
.addScriptField("s_obj2_arr2", "python", "_source['obj2']['arr2']", null)
.execute().actionGet();
Map<String, Object> sObj1 = (Map<String, Object>) response.getHits().getAt(0).field("s_obj1").value();
assertThat(sObj1.get("test").toString(), equalTo("something"));
assertThat(response.getHits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something"));
Map<String, Object> sObj2 = (Map<String, Object>) response.getHits().getAt(0).field("s_obj2").value();
List sObj2Arr2 = (List) sObj2.get("arr2");
assertThat(sObj2Arr2.size(), equalTo(2));
assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1"));
assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2"));
sObj2Arr2 = (List) response.getHits().getAt(0).field("s_obj2_arr2").values();
assertThat(sObj2Arr2.size(), equalTo(2));
assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1"));
assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2"));
}
@Test
public void testCustomScriptBoost() throws Exception {
createIndex("test");
index("test", "type1", "1", jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject());
index("test", "type1", "2", jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject());
refresh();
logger.info("--- QUERY_THEN_FETCH");
logger.info(" --> running doc['num1'].value");
SearchResponse response = client().search(searchRequest()
.searchType(SearchType.QUERY_THEN_FETCH)
.source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value"))
.add(ScoreFunctionBuilders.scriptFunction("doc['num1'].value").lang("python"))))
).actionGet();
assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0));
assertThat(response.getHits().totalHits(), equalTo(2l));
logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation());
logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation());
assertThat(response.getHits().getAt(0).id(), equalTo("2"));
assertThat(response.getHits().getAt(1).id(), equalTo("1"));
logger.info(" --> running -doc['num1'].value");
response = client().search(searchRequest()
.searchType(SearchType.QUERY_THEN_FETCH)
.source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value"))
.add(ScoreFunctionBuilders.scriptFunction("-doc['num1'].value").lang("python"))))
).actionGet();
assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0));
assertThat(response.getHits().totalHits(), equalTo(2l));
logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation());
logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation());
assertThat(response.getHits().getAt(0).id(), equalTo("1"));
assertThat(response.getHits().getAt(1).id(), equalTo("2"));
logger.info(" --> running doc['num1'].value * _score");
response = client().search(searchRequest()
.searchType(SearchType.QUERY_THEN_FETCH)
.source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value"))
.add(ScoreFunctionBuilders.scriptFunction("doc['num1'].value * _score.doubleValue()").lang("python"))))
).actionGet();
assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0));
assertThat(response.getHits().totalHits(), equalTo(2l));
logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation());
logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation());
assertThat(response.getHits().getAt(0).id(), equalTo("2"));
assertThat(response.getHits().getAt(1).id(), equalTo("1"));
logger.info(" --> running param1 * param2 * _score");
response = client().search(searchRequest()
.searchType(SearchType.QUERY_THEN_FETCH)
.source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value"))
.add(ScoreFunctionBuilders.scriptFunction("param1 * param2 * _score.doubleValue()").param("param1", 2).param("param2", 2).lang("python"))))
).actionGet();
assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0));
assertThat(response.getHits().totalHits(), equalTo(2l));
logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation());
logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation());
}
/**
* Test case for #4: https://github.com/elasticsearch/elasticsearch-lang-python/issues/4
* Update request that uses python script with no parameters fails with NullPointerException
* @throws Exception
*/
@Test
public void testPythonEmptyParameters() throws Exception {
createIndex("test");
index("test", "type1", "1", jsonBuilder().startObject().field("myfield", "foo").endObject());
refresh();
client().prepareUpdate("test", "type1", "1").setScriptLang("python")
.setScript("ctx[\"_source\"][\"myfield\"]=\"bar\"", ScriptService.ScriptType.INLINE)
.execute().actionGet();
refresh();
Object value = get("test", "type1", "1").getSourceAsMap().get("myfield");
assertThat(value instanceof String, is(true));
assertThat((String) value, CoreMatchers.equalTo("bar"));
}
@Test
public void testScriptScoresNested() throws IOException {
createIndex("index");
ensureYellow();
index("index", "testtype", "1", jsonBuilder().startObject().field("dummy_field", 1).endObject());
refresh();
SearchResponse response = client().search(
searchRequest().source(
searchSource().query(
functionScoreQuery(
functionScoreQuery(
functionScoreQuery().add(scriptFunction("1").lang("python")))
.add(scriptFunction("_score.doubleValue()").lang("python")))
.add(scriptFunction("_score.doubleValue()").lang("python")
)
)
)
).actionGet();
assertSearchResponse(response);
assertThat(response.getHits().getAt(0).score(), equalTo(1.0f));
}
@Test
public void testScriptScoresWithAgg() throws IOException {
createIndex("index");
ensureYellow();
index("index", "testtype", "1", jsonBuilder().startObject().field("dummy_field", 1).endObject());
refresh();
SearchResponse response = client().search(
searchRequest().source(
searchSource().query(
functionScoreQuery()
.add(scriptFunction("_score.doubleValue()").lang("python")
)
).aggregation(terms("score_agg").script("_score.doubleValue()").lang("python"))
)
).actionGet();
assertSearchResponse(response);
assertThat(response.getHits().getAt(0).score(), equalTo(1.0f));
assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getKeyAsNumber().floatValue(), Matchers.is(1f));
assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getDocCount(), Matchers.is(1l));
}
/**
* Test case for #19: https://github.com/elasticsearch/elasticsearch-lang-python/issues/19
* Multi-line or multi-statement Python scripts raise NullPointerException
*/
@Test
public void testPythonMultiLines() throws Exception {
createIndex("test");
index("test", "type1", "1", jsonBuilder().startObject().field("myfield", "foo").endObject());
refresh();
client().prepareUpdate("test", "type1", "1").setScriptLang("python")
.setScript("a=42; ctx[\"_source\"][\"myfield\"]=\"bar\"", ScriptService.ScriptType.INLINE)
.execute().actionGet();
refresh();
Object value = get("test", "type1", "1").getSourceAsMap().get("myfield");
assertThat(value instanceof String, is(true));
assertThat((String) value, CoreMatchers.equalTo("bar"));
}
}

71
plugins/lang-python/src/test/java/org/elasticsearch/script/python/SimpleBench.java Normal file
View File

@ -0,0 +1,71 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.python;
import org.elasticsearch.common.StopWatch;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.script.ExecutableScript;
import java.util.HashMap;
import java.util.Map;
/**
*
*/
public class SimpleBench {
public static void main(String[] args) {
PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
Object compiled = se.compile("x + y");
Map<String, Object> vars = new HashMap<String, Object>();
// warm up
for (int i = 0; i < 1000; i++) {
vars.put("x", i);
vars.put("y", i + 1);
se.execute(compiled, vars);
}
final long ITER = 100000;
StopWatch stopWatch = new StopWatch().start();
for (long i = 0; i < ITER; i++) {
se.execute(compiled, vars);
}
System.out.println("Execute Took: " + stopWatch.stop().lastTaskTime());
stopWatch = new StopWatch().start();
ExecutableScript executableScript = se.executable(compiled, vars);
for (long i = 0; i < ITER; i++) {
executableScript.run();
}
System.out.println("Executable Took: " + stopWatch.stop().lastTaskTime());
stopWatch = new StopWatch().start();
executableScript = se.executable(compiled, vars);
for (long i = 0; i < ITER; i++) {
for (Map.Entry<String, Object> entry : vars.entrySet()) {
executableScript.setNextVar(entry.getKey(), entry.getValue());
}
executableScript.run();
}
System.out.println("Executable (vars) Took: " + stopWatch.stop().lastTaskTime());
}
}