honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

first commit

This commit is contained in:
Shay Banon 2011-12-05 17:54:04 +02:00
commit 8b5223fcfd
12 changed files with 1107 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.gitignore vendored Normal file
View File

@ -0,0 +1,7 @@
/data
/work
/logs
/.idea
/target
.DS_Store
*.iml

15
README.md Normal file
View File

@ -0,0 +1,15 @@
Python lang Plugin for ElasticSearch
==================================
The Python (jython) language plugin allows to have `python` as the language of scripts to execute.
In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/1.0.0`.
---------------------------------------
| Python Plugin | ElasticSearch |
---------------------------------------
| master | 0.18 -> master |
---------------------------------------
| 1.0.0 | 0.18 -> master |
---------------------------------------

129
pom.xml Normal file
View File

@ -0,0 +1,129 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>elasticsearch-lang-python</name>
<modelVersion>4.0.0</modelVersion>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch-lang-python</artifactId>
<version>1.0.0</version>
<packaging>jar</packaging>
<description>JavaScript lang plugin for ElasticSearch</description>
<inceptionYear>2009</inceptionYear>
<licenses>
<license>
<name>The Apache Software License, Version 2.0</name>
<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
<distribution>repo</distribution>
</license>
</licenses>
<scm>
<connection>scm:git:git@github.com:elasticsearch/elasticsearch-lang-python.git</connection>
<developerConnection>scm:git:git@github.com:elasticsearch/elasticsearch-lang-python.git</developerConnection>
<url>http://github.com/elasticsearch/elasticsearch-lang-python</url>
</scm>
<parent>
<groupId>org.sonatype.oss</groupId>
<artifactId>oss-parent</artifactId>
<version>7</version>
</parent>
<properties>
<elasticsearch.version>0.18.5</elasticsearch.version>
</properties>
<repositories>
</repositories>
<dependencies>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>${elasticsearch.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.python</groupId>
<artifactId>jython-standalone</artifactId>
<version>2.5.2</version>
<scope>compile</scope>
<exclusions>
</exclusions>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.16</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>6.3.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
<version>1.3.RC2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-library</artifactId>
<version>1.3.RC2</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>1.6</source>
<target>1.6</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.11</version>
<configuration>
<includes>
<include>**/*Tests.java</include>
</includes>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>2.1.2</version>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptors>
<descriptor>${basedir}/src/main/assemblies/plugin.xml</descriptor>
</descriptors>
</configuration>
</plugin>
</plugins>
</build>
</project>

26
src/main/assemblies/plugin.xml Normal file
View File

@ -0,0 +1,26 @@
<?xml version="1.0"?>
<assembly>
<id></id>
<formats>
<format>zip</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<dependencySets>
<dependencySet>
<outputDirectory>/</outputDirectory>
<useProjectArtifact>true</useProjectArtifact>
<useTransitiveFiltering>true</useTransitiveFiltering>
<excludes>
<exclude>org.elasticsearch:elasticsearch</exclude>
</excludes>
</dependencySet>
<dependencySet>
<outputDirectory>/</outputDirectory>
<useProjectArtifact>true</useProjectArtifact>
<useTransitiveFiltering>true</useTransitiveFiltering>
<includes>
<include>org.python:jython-standalone</include>
</includes>
</dependencySet>
</dependencySets>
</assembly>

48
src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.plugin.python;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.plugins.AbstractPlugin;
import org.elasticsearch.script.ScriptModule;
import org.elasticsearch.script.python.PythonScriptEngineService;
/**
*
*/
public class PythonPlugin extends AbstractPlugin {
@Override
public String name() {
return "lang-python";
}
@Override
public String description() {
return "Python plugin allowing to add javascript scripting support";
}
@Override
public void processModule(Module module) {
if (module instanceof ScriptModule) {
((ScriptModule) module).addScriptEngine(PythonScriptEngineService.class);
}
}
}

229
src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java Normal file
View File

@ -0,0 +1,229 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.python;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Scorer;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.script.ScriptEngineService;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.lookup.SearchLookup;
import org.python.core.Py;
import org.python.core.PyCode;
import org.python.core.PyObject;
import org.python.core.PyStringMap;
import org.python.util.PythonInterpreter;
import java.util.Map;
/**
*
*/
//TODO we can optimize the case for Map<String, Object> similar to PyStringMap
public class PythonScriptEngineService extends AbstractComponent implements ScriptEngineService {
private final PythonInterpreter interp;
@Inject
public PythonScriptEngineService(Settings settings) {
super(settings);
this.interp = PythonInterpreter.threadLocalStateInterpreter(null);
}
@Override
public String[] types() {
return new String[]{"python", "py"};
}
@Override
public String[] extensions() {
return new String[]{"py"};
}
@Override
public Object compile(String script) {
return interp.compile(script);
}
@Override
public ExecutableScript executable(Object compiledScript, Map<String, Object> vars) {
return new PythonExecutableScript((PyCode) compiledScript, vars);
}
@Override
public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable Map<String, Object> vars) {
return new PythonSearchScript((PyCode) compiledScript, vars, lookup);
}
@Override
public Object execute(Object compiledScript, Map<String, Object> vars) {
PyObject pyVars = Py.java2py(vars);
interp.setLocals(pyVars);
PyObject ret = interp.eval((PyCode) compiledScript);
if (ret == null) {
return null;
}
return ret.__tojava__(Object.class);
}
@Override
public Object unwrap(Object value) {
return unwrapValue(value);
}
@Override
public void close() {
interp.cleanup();
}
public class PythonExecutableScript implements ExecutableScript {
private final PyCode code;
private final PyStringMap pyVars;
public PythonExecutableScript(PyCode code, Map<String, Object> vars) {
this.code = code;
this.pyVars = new PyStringMap();
for (Map.Entry<String, Object> entry : vars.entrySet()) {
pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue()));
}
}
@Override
public void setNextVar(String name, Object value) {
pyVars.__setitem__(name, Py.java2py(value));
}
@Override
public Object run() {
interp.setLocals(pyVars);
PyObject ret = interp.eval(code);
if (ret == null) {
return null;
}
return ret.__tojava__(Object.class);
}
@Override
public Object unwrap(Object value) {
return unwrapValue(value);
}
}
public class PythonSearchScript implements SearchScript {
private final PyCode code;
private final PyStringMap pyVars;
private final SearchLookup lookup;
public PythonSearchScript(PyCode code, Map<String, Object> vars, SearchLookup lookup) {
this.code = code;
this.pyVars = new PyStringMap();
for (Map.Entry<String, Object> entry : lookup.asMap().entrySet()) {
pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue()));
}
if (vars != null) {
for (Map.Entry<String, Object> entry : vars.entrySet()) {
pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue()));
}
}
this.lookup = lookup;
}
@Override
public void setScorer(Scorer scorer) {
lookup.setScorer(scorer);
}
@Override
public void setNextReader(IndexReader reader) {
lookup.setNextReader(reader);
}
@Override
public void setNextDocId(int doc) {
lookup.setNextDocId(doc);
}
@Override
public void setNextSource(Map<String, Object> source) {
lookup.source().setNextSource(source);
}
@Override
public void setNextScore(float score) {
pyVars.__setitem__("_score", Py.java2py(score));
}
@Override
public void setNextVar(String name, Object value) {
pyVars.__setitem__(name, Py.java2py(value));
}
@Override
public Object run() {
interp.setLocals(pyVars);
PyObject ret = interp.eval(code);
if (ret == null) {
return null;
}
return ret.__tojava__(Object.class);
}
@Override
public float runAsFloat() {
return ((Number) run()).floatValue();
}
@Override
public long runAsLong() {
return ((Number) run()).longValue();
}
@Override
public double runAsDouble() {
return ((Number) run()).doubleValue();
}
@Override
public Object unwrap(Object value) {
return unwrapValue(value);
}
}
public static Object unwrapValue(Object value) {
if (value == null) {
return null;
} else if (value instanceof PyObject) {
// seems like this is enough, inner PyDictionary will do the conversion for us for example, so expose it directly
return ((PyObject) value).__tojava__(Object.class);
}
return value;
}
}

1
src/main/resources/es-plugin.properties Normal file
View File

@ -0,0 +1 @@
plugin=org.elasticsearch.plugin.python.PythonPlugin

150
src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java Normal file
View File

@ -0,0 +1,150 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.python;
import org.elasticsearch.common.collect.Lists;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.script.ExecutableScript;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
/**
*
*/
@Test
public class PythonScriptEngineTests {
private PythonScriptEngineService se;
@BeforeClass
public void setup() {
se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS);
}
@AfterClass
public void close() {
se.close();
}
@Test
public void testSimpleEquation() {
Map<String, Object> vars = new HashMap<String, Object>();
Object o = se.execute(se.compile("1 + 2"), vars);
assertThat(((Number) o).intValue(), equalTo(3));
}
@Test
public void testMapAccess() {
Map<String, Object> vars = new HashMap<String, Object>();
Map<String, Object> obj2 = MapBuilder.<String, Object>newMapBuilder().put("prop2", "value2").map();
Map<String, Object> obj1 = MapBuilder.<String, Object>newMapBuilder().put("prop1", "value1").put("obj2", obj2).put("l", Lists.newArrayList("2", "1")).map();
vars.put("obj1", obj1);
Object o = se.execute(se.compile("obj1"), vars);
assertThat(o, instanceOf(Map.class));
obj1 = (Map<String, Object>) o;
assertThat((String) obj1.get("prop1"), equalTo("value1"));
assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
o = se.execute(se.compile("obj1['l'][0]"), vars);
assertThat(((String) o), equalTo("2"));
}
@Test
public void testObjectMapInter() {
Map<String, Object> vars = new HashMap<String, Object>();
Map<String, Object> ctx = new HashMap<String, Object>();
Map<String, Object> obj1 = new HashMap<String, Object>();
obj1.put("prop1", "value1");
ctx.put("obj1", obj1);
vars.put("ctx", ctx);
se.execute(se.compile("ctx['obj2'] = { 'prop2' : 'value2' }; ctx['obj1']['prop1'] = 'uvalue1'"), vars);
ctx = (Map<String, Object>) se.unwrap(vars.get("ctx"));
assertThat(ctx.containsKey("obj1"), equalTo(true));
assertThat((String) ((Map<String, Object>) ctx.get("obj1")).get("prop1"), equalTo("uvalue1"));
assertThat(ctx.containsKey("obj2"), equalTo(true));
assertThat((String) ((Map<String, Object>) ctx.get("obj2")).get("prop2"), equalTo("value2"));
}
@Test
public void testAccessListInScript() {
Map<String, Object> vars = new HashMap<String, Object>();
Map<String, Object> obj2 = MapBuilder.<String, Object>newMapBuilder().put("prop2", "value2").map();
Map<String, Object> obj1 = MapBuilder.<String, Object>newMapBuilder().put("prop1", "value1").put("obj2", obj2).map();
vars.put("l", Lists.newArrayList("1", "2", "3", obj1));
// Object o = se.execute(se.compile("l.length"), vars);
// assertThat(((Number) o).intValue(), equalTo(4));
Object o = se.execute(se.compile("l[0]"), vars);
assertThat(((String) o), equalTo("1"));
o = se.execute(se.compile("l[3]"), vars);
obj1 = (Map<String, Object>) o;
assertThat((String) obj1.get("prop1"), equalTo("value1"));
assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
o = se.execute(se.compile("l[3]['prop1']"), vars);
assertThat(((String) o), equalTo("value1"));
}
@Test
public void testChangingVarsCrossExecution1() {
Map<String, Object> vars = new HashMap<String, Object>();
Map<String, Object> ctx = new HashMap<String, Object>();
vars.put("ctx", ctx);
Object compiledScript = se.compile("ctx['value']");
ExecutableScript script = se.executable(compiledScript, vars);
ctx.put("value", 1);
Object o = script.run();
assertThat(((Number) o).intValue(), equalTo(1));
ctx.put("value", 2);
o = script.run();
assertThat(((Number) o).intValue(), equalTo(2));
}
@Test
public void testChangingVarsCrossExecution2() {
Map<String, Object> vars = new HashMap<String, Object>();
Map<String, Object> ctx = new HashMap<String, Object>();
Object compiledScript = se.compile("value");
ExecutableScript script = se.executable(compiledScript, vars);
script.setNextVar("value", 1);
Object o = script.run();
assertThat(((Number) o).intValue(), equalTo(1));
script.setNextVar("value", 2);
o = script.run();
assertThat(((Number) o).intValue(), equalTo(2));
}
}

173
src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java Normal file
View File

@ -0,0 +1,173 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.python;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.util.concurrent.jsr166y.ThreadLocalRandom;
import org.elasticsearch.script.ExecutableScript;
import org.testng.annotations.Test;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
/**
*
*/
@Test
public class PythonScriptMultiThreadedTest {
protected final ESLogger logger = Loggers.getLogger(getClass());
@Test
public void testExecutableNoRuntimeParams() throws Exception {
final PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS);
final Object compiled = se.compile("x + y");
final AtomicBoolean failed = new AtomicBoolean();
Thread[] threads = new Thread[50];
final CountDownLatch latch = new CountDownLatch(threads.length);
final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1);
for (int i = 0; i < threads.length; i++) {
threads[i] = new Thread(new Runnable() {
@Override
public void run() {
try {
barrier.await();
long x = ThreadLocalRandom.current().nextInt();
long y = ThreadLocalRandom.current().nextInt();
long addition = x + y;
Map<String, Object> vars = new HashMap<String, Object>();
vars.put("x", x);
vars.put("y", y);
ExecutableScript script = se.executable(compiled, vars);
for (int i = 0; i < 100000; i++) {
long result = ((Number) script.run()).longValue();
assertThat(result, equalTo(addition));
}
} catch (Throwable t) {
failed.set(true);
logger.error("failed", t);
} finally {
latch.countDown();
}
}
});
}
for (int i = 0; i < threads.length; i++) {
threads[i].start();
}
barrier.await();
latch.await();
assertThat(failed.get(), equalTo(false));
}
// @Test public void testExecutableWithRuntimeParams() throws Exception {
// final PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS);
// final Object compiled = se.compile("x + y");
// final AtomicBoolean failed = new AtomicBoolean();
//
// Thread[] threads = new Thread[50];
// final CountDownLatch latch = new CountDownLatch(threads.length);
// final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1);
// for (int i = 0; i < threads.length; i++) {
// threads[i] = new Thread(new Runnable() {
// @Override public void run() {
// try {
// barrier.await();
// long x = ThreadLocalRandom.current().nextInt();
// Map<String, Object> vars = new HashMap<String, Object>();
// vars.put("x", x);
// ExecutableScript script = se.executable(compiled, vars);
// Map<String, Object> runtimeVars = new HashMap<String, Object>();
// for (int i = 0; i < 100000; i++) {
// long y = ThreadLocalRandom.current().nextInt();
// long addition = x + y;
// runtimeVars.put("y", y);
// long result = ((Number) script.run(runtimeVars)).longValue();
// assertThat(result, equalTo(addition));
// }
// } catch (Throwable t) {
// failed.set(true);
// logger.error("failed", t);
// } finally {
// latch.countDown();
// }
// }
// });
// }
// for (int i = 0; i < threads.length; i++) {
// threads[i].start();
// }
// barrier.await();
// latch.await();
// assertThat(failed.get(), equalTo(false));
// }
@Test
public void testExecute() throws Exception {
final PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS);
final Object compiled = se.compile("x + y");
final AtomicBoolean failed = new AtomicBoolean();
Thread[] threads = new Thread[50];
final CountDownLatch latch = new CountDownLatch(threads.length);
final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1);
for (int i = 0; i < threads.length; i++) {
threads[i] = new Thread(new Runnable() {
@Override
public void run() {
try {
barrier.await();
Map<String, Object> runtimeVars = new HashMap<String, Object>();
for (int i = 0; i < 100000; i++) {
long x = ThreadLocalRandom.current().nextInt();
long y = ThreadLocalRandom.current().nextInt();
long addition = x + y;
runtimeVars.put("x", x);
runtimeVars.put("y", y);
long result = ((Number) se.execute(compiled, runtimeVars)).longValue();
assertThat(result, equalTo(addition));
}
} catch (Throwable t) {
failed.set(true);
logger.error("failed", t);
} finally {
latch.countDown();
}
}
});
}
for (int i = 0; i < threads.length; i++) {
threads[i].start();
}
barrier.await();
latch.await();
assertThat(failed.get(), equalTo(false));
}
}

253
src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java Normal file
View File

@ -0,0 +1,253 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.python;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.network.NetworkUtils;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.client.Requests.*;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.FilterBuilders.scriptFilter;
import static org.elasticsearch.index.query.QueryBuilders.*;
import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
/**
*
*/
@Test
public class PythonScriptSearchTests {
protected final ESLogger logger = Loggers.getLogger(getClass());
private Node node;
private Client client;
@BeforeMethod
public void createNodes() throws Exception {
node = NodeBuilder.nodeBuilder().settings(ImmutableSettings.settingsBuilder()
.put("path.data", "target/data")
.put("cluster.name", "test-cluster-" + NetworkUtils.getLocalAddress())
.put("gateway.type", "none")
.put("number_of_shards", 1)).node();
client = node.client();
}
@AfterMethod
public void closeNodes() {
client.close();
node.close();
}
@Test
public void testPythonFilter() throws Exception {
client.admin().indices().prepareCreate("test").execute().actionGet();
client.prepareIndex("test", "type1", "1")
.setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject())
.execute().actionGet();
client.admin().indices().prepareFlush().execute().actionGet();
client.prepareIndex("test", "type1", "2")
.setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject())
.execute().actionGet();
client.admin().indices().prepareFlush().execute().actionGet();
client.prepareIndex("test", "type1", "3")
.setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).endObject())
.execute().actionGet();
client.admin().indices().refresh(refreshRequest()).actionGet();
logger.info("running doc['num1'].value > 1");
SearchResponse response = client.prepareSearch()
.setQuery(filteredQuery(matchAllQuery(), scriptFilter("doc['num1'].value > 1").lang("python")))
.addSort("num1", SortOrder.ASC)
.addScriptField("sNum1", "python", "doc['num1'].value", null)
.execute().actionGet();
assertThat(response.hits().totalHits(), equalTo(2l));
assertThat(response.hits().getAt(0).id(), equalTo("2"));
assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0));
assertThat(response.hits().getAt(1).id(), equalTo("3"));
assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(3.0));
logger.info("running doc['num1'].value > param1");
response = client.prepareSearch()
.setQuery(filteredQuery(matchAllQuery(), scriptFilter("doc['num1'].value > param1").lang("python").addParam("param1", 2)))
.addSort("num1", SortOrder.ASC)
.addScriptField("sNum1", "python", "doc['num1'].value", null)
.execute().actionGet();
assertThat(response.hits().totalHits(), equalTo(1l));
assertThat(response.hits().getAt(0).id(), equalTo("3"));
assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(3.0));
logger.info("running doc['num1'].value > param1");
response = client.prepareSearch()
.setQuery(filteredQuery(matchAllQuery(), scriptFilter("doc['num1'].value > param1").lang("python").addParam("param1", -1)))
.addSort("num1", SortOrder.ASC)
.addScriptField("sNum1", "python", "doc['num1'].value", null)
.execute().actionGet();
assertThat(response.hits().totalHits(), equalTo(3l));
assertThat(response.hits().getAt(0).id(), equalTo("1"));
assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0));
assertThat(response.hits().getAt(1).id(), equalTo("2"));
assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0));
assertThat(response.hits().getAt(2).id(), equalTo("3"));
assertThat((Double) response.hits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0));
}
@Test
public void testScriptFieldUsingSource() throws Exception {
client.admin().indices().prepareCreate("test").execute().actionGet();
client.prepareIndex("test", "type1", "1")
.setSource(jsonBuilder().startObject()
.startObject("obj1").field("test", "something").endObject()
.startObject("obj2").startArray("arr2").value("arr_value1").value("arr_value2").endArray().endObject()
.endObject())
.execute().actionGet();
client.admin().indices().refresh(refreshRequest()).actionGet();
SearchResponse response = client.prepareSearch()
.setQuery(matchAllQuery())
.addField("_source.obj1") // we also automatically detect _source in fields
.addScriptField("s_obj1", "python", "_source['obj1']", null)
.addScriptField("s_obj1_test", "python", "_source['obj1']['test']", null)
.addScriptField("s_obj2", "python", "_source['obj2']", null)
.addScriptField("s_obj2_arr2", "python", "_source['obj2']['arr2']", null)
.execute().actionGet();
Map<String, Object> sObj1 = (Map<String, Object>) response.hits().getAt(0).field("_source.obj1").value();
assertThat(sObj1.get("test").toString(), equalTo("something"));
assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something"));
sObj1 = (Map<String, Object>) response.hits().getAt(0).field("s_obj1").value();
assertThat(sObj1.get("test").toString(), equalTo("something"));
assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something"));
Map<String, Object> sObj2 = (Map<String, Object>) response.hits().getAt(0).field("s_obj2").value();
List sObj2Arr2 = (List) sObj2.get("arr2");
assertThat(sObj2Arr2.size(), equalTo(2));
assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1"));
assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2"));
sObj2Arr2 = (List) response.hits().getAt(0).field("s_obj2_arr2").value();
assertThat(sObj2Arr2.size(), equalTo(2));
assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1"));
assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2"));
}
@Test
public void testCustomScriptBoost() throws Exception {
// execute a search before we create an index
try {
client.prepareSearch().setQuery(termQuery("test", "value")).execute().actionGet();
assert false : "should fail";
} catch (Exception e) {
// ignore, no indices
}
try {
client.prepareSearch("test").setQuery(termQuery("test", "value")).execute().actionGet();
assert false : "should fail";
} catch (Exception e) {
// ignore, no indices
}
client.admin().indices().create(createIndexRequest("test")).actionGet();
client.index(indexRequest("test").type("type1").id("1")
.source(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject())).actionGet();
client.index(indexRequest("test").type("type1").id("2")
.source(jsonBuilder().startObject().field("test", "value check").field("num1", 2.0f).endObject())).actionGet();
client.admin().indices().refresh(refreshRequest()).actionGet();
logger.info("--- QUERY_THEN_FETCH");
logger.info("running doc['num1'].value");
SearchResponse response = client.search(searchRequest()
.searchType(SearchType.QUERY_THEN_FETCH)
.source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("doc['num1'].value").lang("python")))
).actionGet();
assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
assertThat(response.hits().totalHits(), equalTo(2l));
logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation());
logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation());
assertThat(response.hits().getAt(0).id(), equalTo("2"));
assertThat(response.hits().getAt(1).id(), equalTo("1"));
logger.info("running -doc['num1'].value");
response = client.search(searchRequest()
.searchType(SearchType.QUERY_THEN_FETCH)
.source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("-doc['num1'].value").lang("python")))
).actionGet();
assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
assertThat(response.hits().totalHits(), equalTo(2l));
logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation());
logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation());
assertThat(response.hits().getAt(0).id(), equalTo("1"));
assertThat(response.hits().getAt(1).id(), equalTo("2"));
logger.info("running doc['num1'].value * _score");
response = client.search(searchRequest()
.searchType(SearchType.QUERY_THEN_FETCH)
.source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("doc['num1'].value * _score").lang("python")))
).actionGet();
assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
assertThat(response.hits().totalHits(), equalTo(2l));
logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation());
logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation());
assertThat(response.hits().getAt(0).id(), equalTo("2"));
assertThat(response.hits().getAt(1).id(), equalTo("1"));
logger.info("running param1 * param2 * _score");
response = client.search(searchRequest()
.searchType(SearchType.QUERY_THEN_FETCH)
.source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("param1 * param2 * _score").param("param1", 2).param("param2", 2).lang("python")))
).actionGet();
assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
assertThat(response.hits().totalHits(), equalTo(2l));
logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation());
logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation());
}
}

71
src/test/java/org/elasticsearch/script/python/SimpleBench.java Normal file
View File

@ -0,0 +1,71 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.python;
import org.elasticsearch.common.StopWatch;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.script.ExecutableScript;
import java.util.HashMap;
import java.util.Map;
/**
*
*/
public class SimpleBench {
public static void main(String[] args) {
PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS);
Object compiled = se.compile("x + y");
Map<String, Object> vars = new HashMap<String, Object>();
// warm up
for (int i = 0; i < 1000; i++) {
vars.put("x", i);
vars.put("y", i + 1);
se.execute(compiled, vars);
}
final long ITER = 100000;
StopWatch stopWatch = new StopWatch().start();
for (long i = 0; i < ITER; i++) {
se.execute(compiled, vars);
}
System.out.println("Execute Took: " + stopWatch.stop().lastTaskTime());
stopWatch = new StopWatch().start();
ExecutableScript executableScript = se.executable(compiled, vars);
for (long i = 0; i < ITER; i++) {
executableScript.run();
}
System.out.println("Executable Took: " + stopWatch.stop().lastTaskTime());
stopWatch = new StopWatch().start();
executableScript = se.executable(compiled, vars);
for (long i = 0; i < ITER; i++) {
for (Map.Entry<String, Object> entry : vars.entrySet()) {
executableScript.setNextVar(entry.getKey(), entry.getValue());
}
executableScript.run();
}
System.out.println("Executable (vars) Took: " + stopWatch.stop().lastTaskTime());
}
}

5
src/test/resources/log4j.properties Normal file
View File

@ -0,0 +1,5 @@
log4j.rootLogger=INFO, out
log4j.appender.out=org.apache.log4j.ConsoleAppender
log4j.appender.out.layout=org.apache.log4j.PatternLayout
log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n