honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Python Plugin: Allow to use python for scripts, closes #418.

This commit is contained in:
kimchy 2010-10-09 20:59:41 +02:00
parent 05e413a7f1
commit a754ebacd4
11 changed files with 930 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.idea/modules.xml
View File

@ -9,6 +9,7 @@
<module fileurl="file://$PROJECT_DIR$/.idea/modules//plugin-cloud-aws.iml" filepath="$PROJECT_DIR$/.idea/modules//plugin-cloud-aws.iml" /> <module fileurl="file://$PROJECT_DIR$/.idea/modules//plugin-cloud-aws.iml" filepath="$PROJECT_DIR$/.idea/modules//plugin-cloud-aws.iml" />
<module fileurl="file://$PROJECT_DIR$/.idea/modules/plugin-lang-groovy.iml" filepath="$PROJECT_DIR$/.idea/modules/plugin-lang-groovy.iml" /> <module fileurl="file://$PROJECT_DIR$/.idea/modules/plugin-lang-groovy.iml" filepath="$PROJECT_DIR$/.idea/modules/plugin-lang-groovy.iml" />
<module fileurl="file://$PROJECT_DIR$/.idea/modules//plugin-lang-javascript.iml" filepath="$PROJECT_DIR$/.idea/modules//plugin-lang-javascript.iml" /> <module fileurl="file://$PROJECT_DIR$/.idea/modules//plugin-lang-javascript.iml" filepath="$PROJECT_DIR$/.idea/modules//plugin-lang-javascript.iml" />
<module fileurl="file://$PROJECT_DIR$/.idea/modules//plugin-lang-python.iml" filepath="$PROJECT_DIR$/.idea/modules//plugin-lang-python.iml" />
<module fileurl="file://$PROJECT_DIR$/.idea/modules//plugin-lang-ruby.iml" filepath="$PROJECT_DIR$/.idea/modules//plugin-lang-ruby.iml" /> <module fileurl="file://$PROJECT_DIR$/.idea/modules//plugin-lang-ruby.iml" filepath="$PROJECT_DIR$/.idea/modules//plugin-lang-ruby.iml" />
<module fileurl="file://$PROJECT_DIR$/.idea/modules//plugin-mapper-attachments.iml" filepath="$PROJECT_DIR$/.idea/modules//plugin-mapper-attachments.iml" /> <module fileurl="file://$PROJECT_DIR$/.idea/modules//plugin-mapper-attachments.iml" filepath="$PROJECT_DIR$/.idea/modules//plugin-mapper-attachments.iml" />
<module fileurl="file://$PROJECT_DIR$/.idea/modules//plugin-river-couchdb.iml" filepath="$PROJECT_DIR$/.idea/modules//plugin-river-couchdb.iml" /> <module fileurl="file://$PROJECT_DIR$/.idea/modules//plugin-river-couchdb.iml" filepath="$PROJECT_DIR$/.idea/modules//plugin-river-couchdb.iml" />

29
.idea/modules/plugin-lang-python.iml Normal file
View File

@ -0,0 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="false">
<output url="file://$MODULE_DIR$/../../plugins/lang/python/build/classes/main" />
<output-test url="file://$MODULE_DIR$/../../plugins/lang/python/build/classes/test" />
<exclude-output />
<content url="file://$MODULE_DIR$/../../plugins/lang/python">
<sourceFolder url="file://$MODULE_DIR$/../../plugins/lang/python/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/../../plugins/lang/python/src/test/java" isTestSource="true" />
<excludeFolder url="file://$MODULE_DIR$/../../plugins/lang/python/build" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="module" module-name="elasticsearch" />
<orderEntry type="module-library">
<library name="jython">
<CLASSES>
<root url="jar://$GRADLE_REPOSITORY$/org.python/jython/jars/jython-2.5.2b2.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module" module-name="test-testng" scope="TEST" />
<orderEntry type="library" scope="TEST" name="testng" level="project" />
<orderEntry type="library" scope="TEST" name="hamcrest" level="project" />
</component>
</module>

145
plugins/lang/python/build.gradle Normal file
View File

@ -0,0 +1,145 @@
dependsOn(':elasticsearch')
apply plugin: 'java'
apply plugin: 'maven'
archivesBaseName = "elasticsearch-lang-python"
explodedDistDir = new File(distsDir, 'exploded')
configurations.compile.transitive = true
configurations.testCompile.transitive = true
// no need to use the resource dir
sourceSets.main.resources.srcDirs 'src/main/java'
sourceSets.test.resources.srcDirs 'src/test/java'
// add the source files to the dist jar
//jar {
// from sourceSets.main.allSource
//}
configurations {
dists
distLib {
visible = false
transitive = false
}
}
dependencies {
compile project(':elasticsearch')
compile('org.python:jython:2.5.2b2') { transitive = false }
distLib('org.python:jython:2.5.2b2') { transitive = false }
testCompile project(':test-testng')
testCompile('org.testng:testng:5.10:jdk15') { transitive = false }
testCompile 'org.hamcrest:hamcrest-all:1.1'
}
test {
useTestNG()
jmvArgs = ["-ea", "-Xmx1024m"]
suiteName = project.name
listeners = ["org.elasticsearch.util.testng.Listeners"]
systemProperties["es.test.log.conf"] = System.getProperty("es.test.log.conf", "log4j-gradle.properties")
}
task explodedDist(dependsOn: [jar], description: 'Builds the plugin zip file') << {
[explodedDistDir]*.mkdirs()
copy {
from configurations.distLib
into explodedDistDir
}
// remove elasticsearch files (compile above adds the elasticsearch one)
ant.delete { fileset(dir: explodedDistDir, includes: "elasticsearch-*.jar") }
copy {
from libsDir
into explodedDistDir
}
ant.delete { fileset(dir: explodedDistDir, includes: "elasticsearch-*-javadoc.jar") }
ant.delete { fileset(dir: explodedDistDir, includes: "elasticsearch-*-sources.jar") }
}
task zip(type: Zip, dependsOn: ['explodedDist']) {
from(explodedDistDir) {
}
}
task release(dependsOn: [zip]) << {
ant.delete(dir: explodedDistDir)
copy {
from distsDir
into(new File(rootProject.distsDir, "plugins"))
}
}
configurations {
deployerJars
}
dependencies {
deployerJars "org.apache.maven.wagon:wagon-http:1.0-beta-2"
}
task sourcesJar(type: Jar, dependsOn: classes) {
classifier = 'sources'
from sourceSets.main.allSource
}
task javadocJar(type: Jar, dependsOn: javadoc) {
classifier = 'javadoc'
from javadoc.destinationDir
}
jar {
// from sourceSets.main.allJava
manifest {
attributes("Implementation-Title": "ElasticSearch", "Implementation-Version": rootProject.version, "Implementation-Date": buildTimeStr)
}
}
artifacts {
archives sourcesJar
archives javadocJar
}
uploadArchives {
repositories.mavenDeployer {
configuration = configurations.deployerJars
repository(url: rootProject.mavenRepoUrl) {
authentication(userName: rootProject.mavenRepoUser, password: rootProject.mavenRepoPass)
}
snapshotRepository(url: rootProject.mavenSnapshotRepoUrl) {
authentication(userName: rootProject.mavenRepoUser, password: rootProject.mavenRepoPass)
}
pom.project {
inceptionYear '2009'
name 'elasticsearch-plugins-lang-python'
description 'Python Plugin for ElasticSearch'
licenses {
license {
name 'The Apache Software License, Version 2.0'
url 'http://www.apache.org/licenses/LICENSE-2.0.txt'
distribution 'repo'
}
}
scm {
connection 'git://github.com/elasticsearch/elasticsearch.git'
developerConnection 'git@github.com:elasticsearch/elasticsearch.git'
url 'http://github.com/elasticsearch/elasticsearch'
}
}
pom.whenConfigured {pom ->
pom.dependencies = pom.dependencies.findAll {dep -> dep.scope != 'test' } // removes the test scoped ones
}
}
}

1
plugins/lang/python/src/main/java/es-plugin.properties Normal file
View File

@ -0,0 +1 @@
plugin=org.elasticsearch.plugin.python.PythonPlugin

45
plugins/lang/python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.plugin.python;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.plugins.AbstractPlugin;
import org.elasticsearch.script.ScriptModule;
import org.elasticsearch.script.python.PythonScriptEngineService;
/**
* @author kimchy (shay.banon)
*/
public class PythonPlugin extends AbstractPlugin {
@Override public String name() {
return "lang-python";
}
@Override public String description() {
return "Python plugin allowing to add javascript scripting support";
}
@Override public void processModule(Module module) {
if (module instanceof ScriptModule) {
((ScriptModule) module).addScriptEngine(PythonScriptEngineService.class);
}
}
}

110
plugins/lang/python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java Normal file
View File

@ -0,0 +1,110 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.python;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.script.ScriptEngineService;
import org.python.core.Py;
import org.python.core.PyCode;
import org.python.core.PyObject;
import org.python.core.PyStringMap;
import org.python.util.PythonInterpreter;
import java.util.Map;
/**
* @author kimchy (shay.banon)
*/
//TODO we can optimize the case for Map<String, Object> similar to PyStringMap
public class PythonScriptEngineService extends AbstractComponent implements ScriptEngineService {
private final PythonInterpreter interp;
@Inject public PythonScriptEngineService(Settings settings) {
super(settings);
this.interp = PythonInterpreter.threadLocalStateInterpreter(null);
}
@Override public String[] types() {
return new String[]{"python"};
}
@Override public Object compile(String script) {
return interp.compile(script);
}
@Override public ExecutableScript executable(Object compiledScript, Map<String, Object> vars) {
return new PythonExecutableScript((PyCode) compiledScript, vars);
}
@Override public Object execute(Object compiledScript, Map<String, Object> vars) {
PyObject pyVars = Py.java2py(vars);
interp.setLocals(pyVars);
PyObject ret = interp.eval((PyCode) compiledScript);
if (ret == null) {
return null;
}
return ret.__tojava__(Object.class);
}
@Override public void close() {
interp.cleanup();
}
public class PythonExecutableScript implements ExecutableScript {
private final PyCode code;
private final PyStringMap pyVars;
public PythonExecutableScript(PyCode code, Map<String, Object> vars) {
this.code = code;
this.pyVars = new PyStringMap();
for (Map.Entry<String, Object> entry : vars.entrySet()) {
pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue()));
}
}
@Override public Object run() {
interp.setLocals(pyVars);
PyObject ret = interp.eval(code);
if (ret == null) {
return null;
}
return ret.__tojava__(Object.class);
}
@Override public Object run(Map<String, Object> vars) {
for (Map.Entry<String, Object> entry : vars.entrySet()) {
pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue()));
}
interp.setLocals(pyVars);
PyObject ret = interp.eval(code);
if (ret == null) {
return null;
}
return ret.__tojava__(Object.class);
}
}
}

119
plugins/lang/python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java Normal file
View File

@ -0,0 +1,119 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.python;
import org.elasticsearch.common.collect.Lists;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.script.ExecutableScript;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.MatcherAssert.*;
import static org.hamcrest.Matchers.*;
/**
* @author kimchy (shay.banon)
*/
public class PythonScriptEngineTests {
private PythonScriptEngineService se;
@BeforeTest public void setup() {
se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS);
}
@Test public void testSimpleEquation() {
Map<String, Object> vars = new HashMap<String, Object>();
Object o = se.execute(se.compile("1 + 2"), vars);
assertThat(((Number) o).intValue(), equalTo(3));
}
@Test public void testMapAccess() {
Map<String, Object> vars = new HashMap<String, Object>();
Map<String, Object> obj2 = MapBuilder.<String, Object>newMapBuilder().put("prop2", "value2").map();
Map<String, Object> obj1 = MapBuilder.<String, Object>newMapBuilder().put("prop1", "value1").put("obj2", obj2).put("l", Lists.newArrayList("2", "1")).map();
vars.put("obj1", obj1);
Object o = se.execute(se.compile("obj1"), vars);
assertThat(o, instanceOf(Map.class));
obj1 = (Map<String, Object>) o;
assertThat((String) obj1.get("prop1"), equalTo("value1"));
assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
o = se.execute(se.compile("obj1['l'][0]"), vars);
assertThat(((String) o), equalTo("2"));
}
@Test public void testAccessListInScript() {
Map<String, Object> vars = new HashMap<String, Object>();
Map<String, Object> obj2 = MapBuilder.<String, Object>newMapBuilder().put("prop2", "value2").map();
Map<String, Object> obj1 = MapBuilder.<String, Object>newMapBuilder().put("prop1", "value1").put("obj2", obj2).map();
vars.put("l", Lists.newArrayList("1", "2", "3", obj1));
// Object o = se.execute(se.compile("l.length"), vars);
// assertThat(((Number) o).intValue(), equalTo(4));
Object o = se.execute(se.compile("l[0]"), vars);
assertThat(((String) o), equalTo("1"));
o = se.execute(se.compile("l[3]"), vars);
obj1 = (Map<String, Object>) o;
assertThat((String) obj1.get("prop1"), equalTo("value1"));
assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
o = se.execute(se.compile("l[3]['prop1']"), vars);
assertThat(((String) o), equalTo("value1"));
}
@Test public void testChangingVarsCrossExecution1() {
Map<String, Object> vars = new HashMap<String, Object>();
Map<String, Object> ctx = new HashMap<String, Object>();
vars.put("ctx", ctx);
Object compiledScript = se.compile("ctx['value']");
ExecutableScript script = se.executable(compiledScript, vars);
ctx.put("value", 1);
Object o = script.run();
assertThat(((Number) o).intValue(), equalTo(1));
ctx.put("value", 2);
o = script.run();
assertThat(((Number) o).intValue(), equalTo(2));
}
@Test public void testChangingVarsCrossExecution2() {
Map<String, Object> vars = new HashMap<String, Object>();
Map<String, Object> ctx = new HashMap<String, Object>();
Object compiledScript = se.compile("value");
ExecutableScript script = se.executable(compiledScript, vars);
ctx.put("value", 1);
Object o = script.run(ctx);
assertThat(((Number) o).intValue(), equalTo(1));
ctx.put("value", 2);
o = script.run(ctx);
assertThat(((Number) o).intValue(), equalTo(2));
}
}

169
plugins/lang/python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java Normal file
View File

@ -0,0 +1,169 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.python;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.util.concurrent.jsr166y.ThreadLocalRandom;
import org.elasticsearch.script.ExecutableScript;
import org.testng.annotations.Test;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.hamcrest.MatcherAssert.*;
import static org.hamcrest.Matchers.*;
/**
* @author kimchy (shay.banon)
*/
@Test
public class PythonScriptMultiThreadedTest {
protected final ESLogger logger = Loggers.getLogger(getClass());
@Test public void testExecutableNoRuntimeParams() throws Exception {
final PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS);
final Object compiled = se.compile("x + y");
final AtomicBoolean failed = new AtomicBoolean();
Thread[] threads = new Thread[50];
final CountDownLatch latch = new CountDownLatch(threads.length);
final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1);
for (int i = 0; i < threads.length; i++) {
threads[i] = new Thread(new Runnable() {
@Override public void run() {
try {
barrier.await();
long x = ThreadLocalRandom.current().nextInt();
long y = ThreadLocalRandom.current().nextInt();
long addition = x + y;
Map<String, Object> vars = new HashMap<String, Object>();
vars.put("x", x);
vars.put("y", y);
ExecutableScript script = se.executable(compiled, vars);
for (int i = 0; i < 100000; i++) {
long result = ((Number) script.run()).longValue();
assertThat(result, equalTo(addition));
}
} catch (Throwable t) {
failed.set(true);
logger.error("failed", t);
} finally {
latch.countDown();
}
}
});
}
for (int i = 0; i < threads.length; i++) {
threads[i].start();
}
barrier.await();
latch.await();
assertThat(failed.get(), equalTo(false));
}
// @Test public void testExecutableWithRuntimeParams() throws Exception {
// final PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS);
// final Object compiled = se.compile("x + y");
// final AtomicBoolean failed = new AtomicBoolean();
//
// Thread[] threads = new Thread[50];
// final CountDownLatch latch = new CountDownLatch(threads.length);
// final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1);
// for (int i = 0; i < threads.length; i++) {
// threads[i] = new Thread(new Runnable() {
// @Override public void run() {
// try {
// barrier.await();
// long x = ThreadLocalRandom.current().nextInt();
// Map<String, Object> vars = new HashMap<String, Object>();
// vars.put("x", x);
// ExecutableScript script = se.executable(compiled, vars);
// Map<String, Object> runtimeVars = new HashMap<String, Object>();
// for (int i = 0; i < 100000; i++) {
// long y = ThreadLocalRandom.current().nextInt();
// long addition = x + y;
// runtimeVars.put("y", y);
// long result = ((Number) script.run(runtimeVars)).longValue();
// assertThat(result, equalTo(addition));
// }
// } catch (Throwable t) {
// failed.set(true);
// logger.error("failed", t);
// } finally {
// latch.countDown();
// }
// }
// });
// }
// for (int i = 0; i < threads.length; i++) {
// threads[i].start();
// }
// barrier.await();
// latch.await();
// assertThat(failed.get(), equalTo(false));
// }
@Test public void testExecute() throws Exception {
final PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS);
final Object compiled = se.compile("x + y");
final AtomicBoolean failed = new AtomicBoolean();
Thread[] threads = new Thread[50];
final CountDownLatch latch = new CountDownLatch(threads.length);
final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1);
for (int i = 0; i < threads.length; i++) {
threads[i] = new Thread(new Runnable() {
@Override public void run() {
try {
barrier.await();
Map<String, Object> runtimeVars = new HashMap<String, Object>();
for (int i = 0; i < 100000; i++) {
long x = ThreadLocalRandom.current().nextInt();
long y = ThreadLocalRandom.current().nextInt();
long addition = x + y;
runtimeVars.put("x", x);
runtimeVars.put("y", y);
long result = ((Number) se.execute(compiled, runtimeVars)).longValue();
assertThat(result, equalTo(addition));
}
} catch (Throwable t) {
failed.set(true);
logger.error("failed", t);
} finally {
latch.countDown();
}
}
});
}
for (int i = 0; i < threads.length; i++) {
threads[i].start();
}
barrier.await();
latch.await();
assertThat(failed.get(), equalTo(false));
}
}

242
plugins/lang/python/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java Normal file
View File

@ -0,0 +1,242 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.python;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.client.Requests.*;
import static org.elasticsearch.common.xcontent.XContentFactory.*;
import static org.elasticsearch.index.query.xcontent.FilterBuilders.*;
import static org.elasticsearch.index.query.xcontent.QueryBuilders.*;
import static org.elasticsearch.search.builder.SearchSourceBuilder.*;
import static org.hamcrest.MatcherAssert.*;
import static org.hamcrest.Matchers.*;
/**
* @author kimchy (shay.banon)
*/
public class PythonScriptSearchTests {
protected final ESLogger logger = Loggers.getLogger(getClass());
private Node node;
private Client client;
@BeforeMethod public void createNodes() throws Exception {
node = NodeBuilder.nodeBuilder().settings(ImmutableSettings.settingsBuilder().put("gateway.type", "none").put("number_of_shards", 1)).node();
client = node.client();
}
@AfterMethod public void closeNodes() {
client.close();
node.close();
}
@Test public void testJavaScriptFilter() throws Exception {
client.admin().indices().prepareCreate("test").execute().actionGet();
client.prepareIndex("test", "type1", "1")
.setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject())
.execute().actionGet();
client.admin().indices().prepareFlush().execute().actionGet();
client.prepareIndex("test", "type1", "2")
.setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject())
.execute().actionGet();
client.admin().indices().prepareFlush().execute().actionGet();
client.prepareIndex("test", "type1", "3")
.setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).endObject())
.execute().actionGet();
client.admin().indices().refresh(refreshRequest()).actionGet();
logger.info("running doc['num1'].value > 1");
SearchResponse response = client.prepareSearch()
.setQuery(filtered(matchAllQuery(), scriptFilter("doc['num1'].value > 1").lang("python")))
.addSort("num1", SortOrder.ASC)
.addScriptField("sNum1", "python", "doc['num1'].value", null)
.execute().actionGet();
assertThat(response.hits().totalHits(), equalTo(2l));
assertThat(response.hits().getAt(0).id(), equalTo("2"));
assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0));
assertThat(response.hits().getAt(1).id(), equalTo("3"));
assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(3.0));
logger.info("running doc['num1'].value > param1");
response = client.prepareSearch()
.setQuery(filtered(matchAllQuery(), scriptFilter("doc['num1'].value > param1").lang("python").addParam("param1", 2)))
.addSort("num1", SortOrder.ASC)
.addScriptField("sNum1", "python", "doc['num1'].value", null)
.execute().actionGet();
assertThat(response.hits().totalHits(), equalTo(1l));
assertThat(response.hits().getAt(0).id(), equalTo("3"));
assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(3.0));
logger.info("running doc['num1'].value > param1");
response = client.prepareSearch()
.setQuery(filtered(matchAllQuery(), scriptFilter("doc['num1'].value > param1").lang("python").addParam("param1", -1)))
.addSort("num1", SortOrder.ASC)
.addScriptField("sNum1", "python", "doc['num1'].value", null)
.execute().actionGet();
assertThat(response.hits().totalHits(), equalTo(3l));
assertThat(response.hits().getAt(0).id(), equalTo("1"));
assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0));
assertThat(response.hits().getAt(1).id(), equalTo("2"));
assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0));
assertThat(response.hits().getAt(2).id(), equalTo("3"));
assertThat((Double) response.hits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0));
}
@Test public void testScriptFieldUsingSource() throws Exception {
client.admin().indices().prepareCreate("test").execute().actionGet();
client.prepareIndex("test", "type1", "1")
.setSource(jsonBuilder().startObject()
.startObject("obj1").field("test", "something").endObject()
.startObject("obj2").startArray("arr2").value("arr_value1").value("arr_value2").endArray().endObject()
.endObject())
.execute().actionGet();
client.admin().indices().refresh(refreshRequest()).actionGet();
SearchResponse response = client.prepareSearch()
.setQuery(matchAllQuery())
.addField("_source.obj1") // we also automatically detect _source in fields
.addScriptField("s_obj1", "python", "_source['obj1']", null)
.addScriptField("s_obj1_test", "python", "_source['obj1']['test']", null)
.addScriptField("s_obj2", "python", "_source['obj2']", null)
.addScriptField("s_obj2_arr2", "python", "_source['obj2']['arr2']", null)
.execute().actionGet();
Map<String, Object> sObj1 = (Map<String, Object>) response.hits().getAt(0).field("_source.obj1").value();
assertThat(sObj1.get("test").toString(), equalTo("something"));
assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something"));
sObj1 = (Map<String, Object>) response.hits().getAt(0).field("s_obj1").value();
assertThat(sObj1.get("test").toString(), equalTo("something"));
assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something"));
Map<String, Object> sObj2 = (Map<String, Object>) response.hits().getAt(0).field("s_obj2").value();
List sObj2Arr2 = (List) sObj2.get("arr2");
assertThat(sObj2Arr2.size(), equalTo(2));
assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1"));
assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2"));
sObj2Arr2 = (List) response.hits().getAt(0).field("s_obj2_arr2").value();
assertThat(sObj2Arr2.size(), equalTo(2));
assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1"));
assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2"));
}
@Test public void testCustomScriptBoost() throws Exception {
// execute a search before we create an index
try {
client.prepareSearch().setQuery(termQuery("test", "value")).execute().actionGet();
assert false : "should fail";
} catch (Exception e) {
// ignore, no indices
}
try {
client.prepareSearch("test").setQuery(termQuery("test", "value")).execute().actionGet();
assert false : "should fail";
} catch (Exception e) {
// ignore, no indices
}
client.admin().indices().create(createIndexRequest("test")).actionGet();
client.index(indexRequest("test").type("type1").id("1")
.source(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject())).actionGet();
client.index(indexRequest("test").type("type1").id("2")
.source(jsonBuilder().startObject().field("test", "value check").field("num1", 2.0f).endObject())).actionGet();
client.admin().indices().refresh(refreshRequest()).actionGet();
logger.info("--- QUERY_THEN_FETCH");
logger.info("running doc['num1'].value");
SearchResponse response = client.search(searchRequest()
.searchType(SearchType.QUERY_THEN_FETCH)
.source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("doc['num1'].value").lang("python")))
).actionGet();
assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
assertThat(response.hits().totalHits(), equalTo(2l));
logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation());
logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation());
assertThat(response.hits().getAt(0).id(), equalTo("2"));
assertThat(response.hits().getAt(1).id(), equalTo("1"));
logger.info("running -doc['num1'].value");
response = client.search(searchRequest()
.searchType(SearchType.QUERY_THEN_FETCH)
.source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("-doc['num1'].value").lang("python")))
).actionGet();
assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
assertThat(response.hits().totalHits(), equalTo(2l));
logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation());
logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation());
assertThat(response.hits().getAt(0).id(), equalTo("1"));
assertThat(response.hits().getAt(1).id(), equalTo("2"));
logger.info("running doc['num1'].value * score");
response = client.search(searchRequest()
.searchType(SearchType.QUERY_THEN_FETCH)
.source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("doc['num1'].value * score").lang("python")))
).actionGet();
assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
assertThat(response.hits().totalHits(), equalTo(2l));
logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation());
logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation());
assertThat(response.hits().getAt(0).id(), equalTo("2"));
assertThat(response.hits().getAt(1).id(), equalTo("1"));
logger.info("running param1 * param2 * score");
response = client.search(searchRequest()
.searchType(SearchType.QUERY_THEN_FETCH)
.source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("param1 * param2 * score").param("param1", 2).param("param2", 2).lang("python")))
).actionGet();
assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
assertThat(response.hits().totalHits(), equalTo(2l));
logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation());
logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation());
}
}

68
plugins/lang/python/src/test/java/org/elasticsearch/script/python/SimpleBench.java Normal file
View File

@ -0,0 +1,68 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.python;
import org.elasticsearch.common.StopWatch;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.script.ExecutableScript;
import java.util.HashMap;
import java.util.Map;
/**
* @author kimchy (shay.banon)
*/
public class SimpleBench {
public static void main(String[] args) {
PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS);
Object compiled = se.compile("x + y");
Map<String, Object> vars = new HashMap<String, Object>();
// warm up
for (int i = 0; i < 1000; i++) {
vars.put("x", i);
vars.put("y", i + 1);
se.execute(compiled, vars);
}
final long ITER = 100000;
StopWatch stopWatch = new StopWatch().start();
for (long i = 0; i < ITER; i++) {
se.execute(compiled, vars);
}
System.out.println("Execute Took: " + stopWatch.stop().lastTaskTime());
stopWatch = new StopWatch().start();
ExecutableScript executableScript = se.executable(compiled, vars);
for (long i = 0; i < ITER; i++) {
executableScript.run();
}
System.out.println("Executable Took: " + stopWatch.stop().lastTaskTime());
stopWatch = new StopWatch().start();
executableScript = se.executable(compiled, vars);
for (long i = 0; i < ITER; i++) {
executableScript.run(vars);
}
System.out.println("Executable (vars) Took: " + stopWatch.stop().lastTaskTime());
}
}

1
settings.gradle
View File

@ -15,6 +15,7 @@ include 'plugins-mapper-attachments'
include 'plugins-lang-groovy' include 'plugins-lang-groovy'
include 'plugins-lang-javascript' include 'plugins-lang-javascript'
include 'plugins-lang-python'
//include 'plugins-lang-ruby' #disabled for now ... //include 'plugins-lang-ruby' #disabled for now ...
include 'plugins-transport-memcached' include 'plugins-transport-memcached'