Python Plugin: Allow to use python for scripts, closes #418.
This commit is contained in:
kimchy
parent
05e413a7f1
commit
a754ebacd4
|
|
@ -9,6 +9,7 @@
|
|||
<module fileurl="file://$PROJECT_DIR$/.idea/modules//plugin-cloud-aws.iml" filepath="$PROJECT_DIR$/.idea/modules//plugin-cloud-aws.iml" />
|
||||
<module fileurl="file://$PROJECT_DIR$/.idea/modules/plugin-lang-groovy.iml" filepath="$PROJECT_DIR$/.idea/modules/plugin-lang-groovy.iml" />
|
||||
<module fileurl="file://$PROJECT_DIR$/.idea/modules//plugin-lang-javascript.iml" filepath="$PROJECT_DIR$/.idea/modules//plugin-lang-javascript.iml" />
|
||||
<module fileurl="file://$PROJECT_DIR$/.idea/modules//plugin-lang-python.iml" filepath="$PROJECT_DIR$/.idea/modules//plugin-lang-python.iml" />
|
||||
<module fileurl="file://$PROJECT_DIR$/.idea/modules//plugin-lang-ruby.iml" filepath="$PROJECT_DIR$/.idea/modules//plugin-lang-ruby.iml" />
|
||||
<module fileurl="file://$PROJECT_DIR$/.idea/modules//plugin-mapper-attachments.iml" filepath="$PROJECT_DIR$/.idea/modules//plugin-mapper-attachments.iml" />
|
||||
<module fileurl="file://$PROJECT_DIR$/.idea/modules//plugin-river-couchdb.iml" filepath="$PROJECT_DIR$/.idea/modules//plugin-river-couchdb.iml" />
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="JAVA_MODULE" version="4">
|
||||
<component name="NewModuleRootManager" inherit-compiler-output="false">
|
||||
<output url="file://$MODULE_DIR$/../../plugins/lang/python/build/classes/main" />
|
||||
<output-test url="file://$MODULE_DIR$/../../plugins/lang/python/build/classes/test" />
|
||||
<exclude-output />
|
||||
<content url="file://$MODULE_DIR$/../../plugins/lang/python">
|
||||
<sourceFolder url="file://$MODULE_DIR$/../../plugins/lang/python/src/main/java" isTestSource="false" />
|
||||
<sourceFolder url="file://$MODULE_DIR$/../../plugins/lang/python/src/test/java" isTestSource="true" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/../../plugins/lang/python/build" />
|
||||
</content>
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
<orderEntry type="module" module-name="elasticsearch" />
|
||||
<orderEntry type="module-library">
|
||||
<library name="jython">
|
||||
<CLASSES>
|
||||
<root url="jar://$GRADLE_REPOSITORY$/org.python/jython/jars/jython-2.5.2b2.jar!/" />
|
||||
</CLASSES>
|
||||
<JAVADOC />
|
||||
<SOURCES />
|
||||
</library>
|
||||
</orderEntry>
|
||||
<orderEntry type="module" module-name="test-testng" scope="TEST" />
|
||||
<orderEntry type="library" scope="TEST" name="testng" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="hamcrest" level="project" />
|
||||
</component>
|
||||
</module>
|
||||
|
||||
|
|
@ -0,0 +1,145 @@
|
|||
dependsOn(':elasticsearch')
|
||||
|
||||
apply plugin: 'java'
|
||||
apply plugin: 'maven'
|
||||
|
||||
archivesBaseName = "elasticsearch-lang-python"
|
||||
|
||||
explodedDistDir = new File(distsDir, 'exploded')
|
||||
|
||||
configurations.compile.transitive = true
|
||||
configurations.testCompile.transitive = true
|
||||
|
||||
// no need to use the resource dir
|
||||
sourceSets.main.resources.srcDirs 'src/main/java'
|
||||
sourceSets.test.resources.srcDirs 'src/test/java'
|
||||
|
||||
// add the source files to the dist jar
|
||||
//jar {
|
||||
// from sourceSets.main.allSource
|
||||
//}
|
||||
|
||||
configurations {
|
||||
dists
|
||||
distLib {
|
||||
visible = false
|
||||
transitive = false
|
||||
}
|
||||
}
|
||||
|
||||
dependencies {
|
||||
compile project(':elasticsearch')
|
||||
|
||||
compile('org.python:jython:2.5.2b2') { transitive = false }
|
||||
distLib('org.python:jython:2.5.2b2') { transitive = false }
|
||||
|
||||
testCompile project(':test-testng')
|
||||
testCompile('org.testng:testng:5.10:jdk15') { transitive = false }
|
||||
testCompile 'org.hamcrest:hamcrest-all:1.1'
|
||||
}
|
||||
|
||||
test {
|
||||
useTestNG()
|
||||
jmvArgs = ["-ea", "-Xmx1024m"]
|
||||
suiteName = project.name
|
||||
listeners = ["org.elasticsearch.util.testng.Listeners"]
|
||||
systemProperties["es.test.log.conf"] = System.getProperty("es.test.log.conf", "log4j-gradle.properties")
|
||||
}
|
||||
|
||||
task explodedDist(dependsOn: [jar], description: 'Builds the plugin zip file') << {
|
||||
[explodedDistDir]*.mkdirs()
|
||||
|
||||
copy {
|
||||
from configurations.distLib
|
||||
into explodedDistDir
|
||||
}
|
||||
|
||||
// remove elasticsearch files (compile above adds the elasticsearch one)
|
||||
ant.delete { fileset(dir: explodedDistDir, includes: "elasticsearch-*.jar") }
|
||||
|
||||
copy {
|
||||
from libsDir
|
||||
into explodedDistDir
|
||||
}
|
||||
|
||||
ant.delete { fileset(dir: explodedDistDir, includes: "elasticsearch-*-javadoc.jar") }
|
||||
ant.delete { fileset(dir: explodedDistDir, includes: "elasticsearch-*-sources.jar") }
|
||||
}
|
||||
|
||||
task zip(type: Zip, dependsOn: ['explodedDist']) {
|
||||
from(explodedDistDir) {
|
||||
}
|
||||
}
|
||||
|
||||
task release(dependsOn: [zip]) << {
|
||||
ant.delete(dir: explodedDistDir)
|
||||
copy {
|
||||
from distsDir
|
||||
into(new File(rootProject.distsDir, "plugins"))
|
||||
}
|
||||
}
|
||||
|
||||
configurations {
|
||||
deployerJars
|
||||
}
|
||||
|
||||
dependencies {
|
||||
deployerJars "org.apache.maven.wagon:wagon-http:1.0-beta-2"
|
||||
}
|
||||
|
||||
task sourcesJar(type: Jar, dependsOn: classes) {
|
||||
classifier = 'sources'
|
||||
from sourceSets.main.allSource
|
||||
}
|
||||
|
||||
task javadocJar(type: Jar, dependsOn: javadoc) {
|
||||
classifier = 'javadoc'
|
||||
from javadoc.destinationDir
|
||||
}
|
||||
|
||||
jar {
|
||||
// from sourceSets.main.allJava
|
||||
manifest {
|
||||
attributes("Implementation-Title": "ElasticSearch", "Implementation-Version": rootProject.version, "Implementation-Date": buildTimeStr)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
artifacts {
|
||||
archives sourcesJar
|
||||
archives javadocJar
|
||||
}
|
||||
|
||||
uploadArchives {
|
||||
repositories.mavenDeployer {
|
||||
configuration = configurations.deployerJars
|
||||
repository(url: rootProject.mavenRepoUrl) {
|
||||
authentication(userName: rootProject.mavenRepoUser, password: rootProject.mavenRepoPass)
|
||||
}
|
||||
snapshotRepository(url: rootProject.mavenSnapshotRepoUrl) {
|
||||
authentication(userName: rootProject.mavenRepoUser, password: rootProject.mavenRepoPass)
|
||||
}
|
||||
|
||||
pom.project {
|
||||
inceptionYear '2009'
|
||||
name 'elasticsearch-plugins-lang-python'
|
||||
description 'Python Plugin for ElasticSearch'
|
||||
licenses {
|
||||
license {
|
||||
name 'The Apache Software License, Version 2.0'
|
||||
url 'http://www.apache.org/licenses/LICENSE-2.0.txt'
|
||||
distribution 'repo'
|
||||
}
|
||||
}
|
||||
scm {
|
||||
connection 'git://github.com/elasticsearch/elasticsearch.git'
|
||||
developerConnection 'git@github.com:elasticsearch/elasticsearch.git'
|
||||
url 'http://github.com/elasticsearch/elasticsearch'
|
||||
}
|
||||
}
|
||||
|
||||
pom.whenConfigured {pom ->
|
||||
pom.dependencies = pom.dependencies.findAll {dep -> dep.scope != 'test' } // removes the test scoped ones
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
plugin=org.elasticsearch.plugin.python.PythonPlugin
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
/*
|
||||
* Licensed to Elastic Search and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. Elastic Search licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.plugin.python;
|
||||
|
||||
import org.elasticsearch.common.inject.Module;
|
||||
import org.elasticsearch.plugins.AbstractPlugin;
|
||||
import org.elasticsearch.script.ScriptModule;
|
||||
import org.elasticsearch.script.python.PythonScriptEngineService;
|
||||
|
||||
/**
|
||||
* @author kimchy (shay.banon)
|
||||
*/
|
||||
public class PythonPlugin extends AbstractPlugin {
|
||||
|
||||
@Override public String name() {
|
||||
return "lang-python";
|
||||
}
|
||||
|
||||
@Override public String description() {
|
||||
return "Python plugin allowing to add javascript scripting support";
|
||||
}
|
||||
|
||||
@Override public void processModule(Module module) {
|
||||
if (module instanceof ScriptModule) {
|
||||
((ScriptModule) module).addScriptEngine(PythonScriptEngineService.class);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,110 @@
|
|||
/*
|
||||
* Licensed to Elastic Search and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. Elastic Search licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.script.python;
|
||||
|
||||
import org.elasticsearch.common.component.AbstractComponent;
|
||||
import org.elasticsearch.common.inject.Inject;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.script.ExecutableScript;
|
||||
import org.elasticsearch.script.ScriptEngineService;
|
||||
import org.python.core.Py;
|
||||
import org.python.core.PyCode;
|
||||
import org.python.core.PyObject;
|
||||
import org.python.core.PyStringMap;
|
||||
import org.python.util.PythonInterpreter;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* @author kimchy (shay.banon)
|
||||
*/
|
||||
//TODO we can optimize the case for Map<String, Object> similar to PyStringMap
|
||||
public class PythonScriptEngineService extends AbstractComponent implements ScriptEngineService {
|
||||
|
||||
private final PythonInterpreter interp;
|
||||
|
||||
@Inject public PythonScriptEngineService(Settings settings) {
|
||||
super(settings);
|
||||
|
||||
this.interp = PythonInterpreter.threadLocalStateInterpreter(null);
|
||||
}
|
||||
|
||||
@Override public String[] types() {
|
||||
return new String[]{"python"};
|
||||
}
|
||||
|
||||
@Override public Object compile(String script) {
|
||||
return interp.compile(script);
|
||||
}
|
||||
|
||||
@Override public ExecutableScript executable(Object compiledScript, Map<String, Object> vars) {
|
||||
return new PythonExecutableScript((PyCode) compiledScript, vars);
|
||||
}
|
||||
|
||||
@Override public Object execute(Object compiledScript, Map<String, Object> vars) {
|
||||
PyObject pyVars = Py.java2py(vars);
|
||||
interp.setLocals(pyVars);
|
||||
PyObject ret = interp.eval((PyCode) compiledScript);
|
||||
if (ret == null) {
|
||||
return null;
|
||||
}
|
||||
return ret.__tojava__(Object.class);
|
||||
}
|
||||
|
||||
@Override public void close() {
|
||||
interp.cleanup();
|
||||
}
|
||||
|
||||
public class PythonExecutableScript implements ExecutableScript {
|
||||
|
||||
private final PyCode code;
|
||||
|
||||
private final PyStringMap pyVars;
|
||||
|
||||
public PythonExecutableScript(PyCode code, Map<String, Object> vars) {
|
||||
this.code = code;
|
||||
this.pyVars = new PyStringMap();
|
||||
for (Map.Entry<String, Object> entry : vars.entrySet()) {
|
||||
pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue()));
|
||||
}
|
||||
}
|
||||
|
||||
@Override public Object run() {
|
||||
interp.setLocals(pyVars);
|
||||
PyObject ret = interp.eval(code);
|
||||
if (ret == null) {
|
||||
return null;
|
||||
}
|
||||
return ret.__tojava__(Object.class);
|
||||
}
|
||||
|
||||
@Override public Object run(Map<String, Object> vars) {
|
||||
for (Map.Entry<String, Object> entry : vars.entrySet()) {
|
||||
pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue()));
|
||||
}
|
||||
interp.setLocals(pyVars);
|
||||
PyObject ret = interp.eval(code);
|
||||
if (ret == null) {
|
||||
return null;
|
||||
}
|
||||
return ret.__tojava__(Object.class);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
/*
|
||||
* Licensed to Elastic Search and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. Elastic Search licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.script.python;
|
||||
|
||||
import org.elasticsearch.common.collect.Lists;
|
||||
import org.elasticsearch.common.collect.MapBuilder;
|
||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.script.ExecutableScript;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.hamcrest.MatcherAssert.*;
|
||||
import static org.hamcrest.Matchers.*;
|
||||
|
||||
/**
|
||||
* @author kimchy (shay.banon)
|
||||
*/
|
||||
public class PythonScriptEngineTests {
|
||||
|
||||
private PythonScriptEngineService se;
|
||||
|
||||
@BeforeTest public void setup() {
|
||||
se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS);
|
||||
}
|
||||
|
||||
@Test public void testSimpleEquation() {
|
||||
Map<String, Object> vars = new HashMap<String, Object>();
|
||||
Object o = se.execute(se.compile("1 + 2"), vars);
|
||||
assertThat(((Number) o).intValue(), equalTo(3));
|
||||
}
|
||||
|
||||
@Test public void testMapAccess() {
|
||||
Map<String, Object> vars = new HashMap<String, Object>();
|
||||
|
||||
Map<String, Object> obj2 = MapBuilder.<String, Object>newMapBuilder().put("prop2", "value2").map();
|
||||
Map<String, Object> obj1 = MapBuilder.<String, Object>newMapBuilder().put("prop1", "value1").put("obj2", obj2).put("l", Lists.newArrayList("2", "1")).map();
|
||||
vars.put("obj1", obj1);
|
||||
Object o = se.execute(se.compile("obj1"), vars);
|
||||
assertThat(o, instanceOf(Map.class));
|
||||
obj1 = (Map<String, Object>) o;
|
||||
assertThat((String) obj1.get("prop1"), equalTo("value1"));
|
||||
assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
|
||||
|
||||
o = se.execute(se.compile("obj1['l'][0]"), vars);
|
||||
assertThat(((String) o), equalTo("2"));
|
||||
}
|
||||
|
||||
@Test public void testAccessListInScript() {
|
||||
Map<String, Object> vars = new HashMap<String, Object>();
|
||||
Map<String, Object> obj2 = MapBuilder.<String, Object>newMapBuilder().put("prop2", "value2").map();
|
||||
Map<String, Object> obj1 = MapBuilder.<String, Object>newMapBuilder().put("prop1", "value1").put("obj2", obj2).map();
|
||||
vars.put("l", Lists.newArrayList("1", "2", "3", obj1));
|
||||
|
||||
// Object o = se.execute(se.compile("l.length"), vars);
|
||||
// assertThat(((Number) o).intValue(), equalTo(4));
|
||||
|
||||
Object o = se.execute(se.compile("l[0]"), vars);
|
||||
assertThat(((String) o), equalTo("1"));
|
||||
|
||||
o = se.execute(se.compile("l[3]"), vars);
|
||||
obj1 = (Map<String, Object>) o;
|
||||
assertThat((String) obj1.get("prop1"), equalTo("value1"));
|
||||
assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
|
||||
|
||||
o = se.execute(se.compile("l[3]['prop1']"), vars);
|
||||
assertThat(((String) o), equalTo("value1"));
|
||||
}
|
||||
|
||||
@Test public void testChangingVarsCrossExecution1() {
|
||||
Map<String, Object> vars = new HashMap<String, Object>();
|
||||
Map<String, Object> ctx = new HashMap<String, Object>();
|
||||
vars.put("ctx", ctx);
|
||||
Object compiledScript = se.compile("ctx['value']");
|
||||
|
||||
ExecutableScript script = se.executable(compiledScript, vars);
|
||||
ctx.put("value", 1);
|
||||
Object o = script.run();
|
||||
assertThat(((Number) o).intValue(), equalTo(1));
|
||||
|
||||
ctx.put("value", 2);
|
||||
o = script.run();
|
||||
assertThat(((Number) o).intValue(), equalTo(2));
|
||||
}
|
||||
|
||||
@Test public void testChangingVarsCrossExecution2() {
|
||||
Map<String, Object> vars = new HashMap<String, Object>();
|
||||
Map<String, Object> ctx = new HashMap<String, Object>();
|
||||
Object compiledScript = se.compile("value");
|
||||
|
||||
ExecutableScript script = se.executable(compiledScript, vars);
|
||||
ctx.put("value", 1);
|
||||
Object o = script.run(ctx);
|
||||
assertThat(((Number) o).intValue(), equalTo(1));
|
||||
|
||||
ctx.put("value", 2);
|
||||
o = script.run(ctx);
|
||||
assertThat(((Number) o).intValue(), equalTo(2));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,169 @@
|
|||
/*
|
||||
* Licensed to Elastic Search and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. Elastic Search licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.script.python;
|
||||
|
||||
import org.elasticsearch.common.logging.ESLogger;
|
||||
import org.elasticsearch.common.logging.Loggers;
|
||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.common.util.concurrent.jsr166y.ThreadLocalRandom;
|
||||
import org.elasticsearch.script.ExecutableScript;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.CyclicBarrier;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import static org.hamcrest.MatcherAssert.*;
|
||||
import static org.hamcrest.Matchers.*;
|
||||
|
||||
/**
|
||||
* @author kimchy (shay.banon)
|
||||
*/
|
||||
@Test
|
||||
public class PythonScriptMultiThreadedTest {
|
||||
|
||||
protected final ESLogger logger = Loggers.getLogger(getClass());
|
||||
|
||||
@Test public void testExecutableNoRuntimeParams() throws Exception {
|
||||
final PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS);
|
||||
final Object compiled = se.compile("x + y");
|
||||
final AtomicBoolean failed = new AtomicBoolean();
|
||||
|
||||
Thread[] threads = new Thread[50];
|
||||
final CountDownLatch latch = new CountDownLatch(threads.length);
|
||||
final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1);
|
||||
for (int i = 0; i < threads.length; i++) {
|
||||
threads[i] = new Thread(new Runnable() {
|
||||
@Override public void run() {
|
||||
try {
|
||||
barrier.await();
|
||||
long x = ThreadLocalRandom.current().nextInt();
|
||||
long y = ThreadLocalRandom.current().nextInt();
|
||||
long addition = x + y;
|
||||
Map<String, Object> vars = new HashMap<String, Object>();
|
||||
vars.put("x", x);
|
||||
vars.put("y", y);
|
||||
ExecutableScript script = se.executable(compiled, vars);
|
||||
for (int i = 0; i < 100000; i++) {
|
||||
long result = ((Number) script.run()).longValue();
|
||||
assertThat(result, equalTo(addition));
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
failed.set(true);
|
||||
logger.error("failed", t);
|
||||
} finally {
|
||||
latch.countDown();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
for (int i = 0; i < threads.length; i++) {
|
||||
threads[i].start();
|
||||
}
|
||||
barrier.await();
|
||||
latch.await();
|
||||
assertThat(failed.get(), equalTo(false));
|
||||
}
|
||||
|
||||
|
||||
// @Test public void testExecutableWithRuntimeParams() throws Exception {
|
||||
// final PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS);
|
||||
// final Object compiled = se.compile("x + y");
|
||||
// final AtomicBoolean failed = new AtomicBoolean();
|
||||
//
|
||||
// Thread[] threads = new Thread[50];
|
||||
// final CountDownLatch latch = new CountDownLatch(threads.length);
|
||||
// final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1);
|
||||
// for (int i = 0; i < threads.length; i++) {
|
||||
// threads[i] = new Thread(new Runnable() {
|
||||
// @Override public void run() {
|
||||
// try {
|
||||
// barrier.await();
|
||||
// long x = ThreadLocalRandom.current().nextInt();
|
||||
// Map<String, Object> vars = new HashMap<String, Object>();
|
||||
// vars.put("x", x);
|
||||
// ExecutableScript script = se.executable(compiled, vars);
|
||||
// Map<String, Object> runtimeVars = new HashMap<String, Object>();
|
||||
// for (int i = 0; i < 100000; i++) {
|
||||
// long y = ThreadLocalRandom.current().nextInt();
|
||||
// long addition = x + y;
|
||||
// runtimeVars.put("y", y);
|
||||
// long result = ((Number) script.run(runtimeVars)).longValue();
|
||||
// assertThat(result, equalTo(addition));
|
||||
// }
|
||||
// } catch (Throwable t) {
|
||||
// failed.set(true);
|
||||
// logger.error("failed", t);
|
||||
// } finally {
|
||||
// latch.countDown();
|
||||
// }
|
||||
// }
|
||||
// });
|
||||
// }
|
||||
// for (int i = 0; i < threads.length; i++) {
|
||||
// threads[i].start();
|
||||
// }
|
||||
// barrier.await();
|
||||
// latch.await();
|
||||
// assertThat(failed.get(), equalTo(false));
|
||||
// }
|
||||
|
||||
@Test public void testExecute() throws Exception {
|
||||
final PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS);
|
||||
final Object compiled = se.compile("x + y");
|
||||
final AtomicBoolean failed = new AtomicBoolean();
|
||||
|
||||
Thread[] threads = new Thread[50];
|
||||
final CountDownLatch latch = new CountDownLatch(threads.length);
|
||||
final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1);
|
||||
for (int i = 0; i < threads.length; i++) {
|
||||
threads[i] = new Thread(new Runnable() {
|
||||
@Override public void run() {
|
||||
try {
|
||||
barrier.await();
|
||||
Map<String, Object> runtimeVars = new HashMap<String, Object>();
|
||||
for (int i = 0; i < 100000; i++) {
|
||||
long x = ThreadLocalRandom.current().nextInt();
|
||||
long y = ThreadLocalRandom.current().nextInt();
|
||||
long addition = x + y;
|
||||
runtimeVars.put("x", x);
|
||||
runtimeVars.put("y", y);
|
||||
long result = ((Number) se.execute(compiled, runtimeVars)).longValue();
|
||||
assertThat(result, equalTo(addition));
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
failed.set(true);
|
||||
logger.error("failed", t);
|
||||
} finally {
|
||||
latch.countDown();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
for (int i = 0; i < threads.length; i++) {
|
||||
threads[i].start();
|
||||
}
|
||||
barrier.await();
|
||||
latch.await();
|
||||
assertThat(failed.get(), equalTo(false));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,242 @@
|
|||
/*
|
||||
* Licensed to Elastic Search and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. Elastic Search licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.script.python;
|
||||
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.action.search.SearchType;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.common.logging.ESLogger;
|
||||
import org.elasticsearch.common.logging.Loggers;
|
||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.node.Node;
|
||||
import org.elasticsearch.node.NodeBuilder;
|
||||
import org.elasticsearch.search.sort.SortOrder;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.elasticsearch.client.Requests.*;
|
||||
import static org.elasticsearch.common.xcontent.XContentFactory.*;
|
||||
import static org.elasticsearch.index.query.xcontent.FilterBuilders.*;
|
||||
import static org.elasticsearch.index.query.xcontent.QueryBuilders.*;
|
||||
import static org.elasticsearch.search.builder.SearchSourceBuilder.*;
|
||||
import static org.hamcrest.MatcherAssert.*;
|
||||
import static org.hamcrest.Matchers.*;
|
||||
|
||||
/**
|
||||
* @author kimchy (shay.banon)
|
||||
*/
|
||||
public class PythonScriptSearchTests {
|
||||
|
||||
protected final ESLogger logger = Loggers.getLogger(getClass());
|
||||
|
||||
private Node node;
|
||||
|
||||
private Client client;
|
||||
|
||||
@BeforeMethod public void createNodes() throws Exception {
|
||||
node = NodeBuilder.nodeBuilder().settings(ImmutableSettings.settingsBuilder().put("gateway.type", "none").put("number_of_shards", 1)).node();
|
||||
client = node.client();
|
||||
}
|
||||
|
||||
@AfterMethod public void closeNodes() {
|
||||
client.close();
|
||||
node.close();
|
||||
}
|
||||
|
||||
@Test public void testJavaScriptFilter() throws Exception {
|
||||
client.admin().indices().prepareCreate("test").execute().actionGet();
|
||||
client.prepareIndex("test", "type1", "1")
|
||||
.setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject())
|
||||
.execute().actionGet();
|
||||
client.admin().indices().prepareFlush().execute().actionGet();
|
||||
client.prepareIndex("test", "type1", "2")
|
||||
.setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject())
|
||||
.execute().actionGet();
|
||||
client.admin().indices().prepareFlush().execute().actionGet();
|
||||
client.prepareIndex("test", "type1", "3")
|
||||
.setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).endObject())
|
||||
.execute().actionGet();
|
||||
client.admin().indices().refresh(refreshRequest()).actionGet();
|
||||
|
||||
logger.info("running doc['num1'].value > 1");
|
||||
SearchResponse response = client.prepareSearch()
|
||||
.setQuery(filtered(matchAllQuery(), scriptFilter("doc['num1'].value > 1").lang("python")))
|
||||
.addSort("num1", SortOrder.ASC)
|
||||
.addScriptField("sNum1", "python", "doc['num1'].value", null)
|
||||
.execute().actionGet();
|
||||
|
||||
assertThat(response.hits().totalHits(), equalTo(2l));
|
||||
assertThat(response.hits().getAt(0).id(), equalTo("2"));
|
||||
assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0));
|
||||
assertThat(response.hits().getAt(1).id(), equalTo("3"));
|
||||
assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(3.0));
|
||||
|
||||
logger.info("running doc['num1'].value > param1");
|
||||
response = client.prepareSearch()
|
||||
.setQuery(filtered(matchAllQuery(), scriptFilter("doc['num1'].value > param1").lang("python").addParam("param1", 2)))
|
||||
.addSort("num1", SortOrder.ASC)
|
||||
.addScriptField("sNum1", "python", "doc['num1'].value", null)
|
||||
.execute().actionGet();
|
||||
|
||||
assertThat(response.hits().totalHits(), equalTo(1l));
|
||||
assertThat(response.hits().getAt(0).id(), equalTo("3"));
|
||||
assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(3.0));
|
||||
|
||||
logger.info("running doc['num1'].value > param1");
|
||||
response = client.prepareSearch()
|
||||
.setQuery(filtered(matchAllQuery(), scriptFilter("doc['num1'].value > param1").lang("python").addParam("param1", -1)))
|
||||
.addSort("num1", SortOrder.ASC)
|
||||
.addScriptField("sNum1", "python", "doc['num1'].value", null)
|
||||
.execute().actionGet();
|
||||
|
||||
assertThat(response.hits().totalHits(), equalTo(3l));
|
||||
assertThat(response.hits().getAt(0).id(), equalTo("1"));
|
||||
assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0));
|
||||
assertThat(response.hits().getAt(1).id(), equalTo("2"));
|
||||
assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0));
|
||||
assertThat(response.hits().getAt(2).id(), equalTo("3"));
|
||||
assertThat((Double) response.hits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0));
|
||||
}
|
||||
|
||||
@Test public void testScriptFieldUsingSource() throws Exception {
|
||||
client.admin().indices().prepareCreate("test").execute().actionGet();
|
||||
client.prepareIndex("test", "type1", "1")
|
||||
.setSource(jsonBuilder().startObject()
|
||||
.startObject("obj1").field("test", "something").endObject()
|
||||
.startObject("obj2").startArray("arr2").value("arr_value1").value("arr_value2").endArray().endObject()
|
||||
.endObject())
|
||||
.execute().actionGet();
|
||||
client.admin().indices().refresh(refreshRequest()).actionGet();
|
||||
|
||||
SearchResponse response = client.prepareSearch()
|
||||
.setQuery(matchAllQuery())
|
||||
.addField("_source.obj1") // we also automatically detect _source in fields
|
||||
.addScriptField("s_obj1", "python", "_source['obj1']", null)
|
||||
.addScriptField("s_obj1_test", "python", "_source['obj1']['test']", null)
|
||||
.addScriptField("s_obj2", "python", "_source['obj2']", null)
|
||||
.addScriptField("s_obj2_arr2", "python", "_source['obj2']['arr2']", null)
|
||||
.execute().actionGet();
|
||||
|
||||
Map<String, Object> sObj1 = (Map<String, Object>) response.hits().getAt(0).field("_source.obj1").value();
|
||||
assertThat(sObj1.get("test").toString(), equalTo("something"));
|
||||
assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something"));
|
||||
|
||||
sObj1 = (Map<String, Object>) response.hits().getAt(0).field("s_obj1").value();
|
||||
assertThat(sObj1.get("test").toString(), equalTo("something"));
|
||||
assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something"));
|
||||
|
||||
Map<String, Object> sObj2 = (Map<String, Object>) response.hits().getAt(0).field("s_obj2").value();
|
||||
List sObj2Arr2 = (List) sObj2.get("arr2");
|
||||
assertThat(sObj2Arr2.size(), equalTo(2));
|
||||
assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1"));
|
||||
assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2"));
|
||||
|
||||
sObj2Arr2 = (List) response.hits().getAt(0).field("s_obj2_arr2").value();
|
||||
assertThat(sObj2Arr2.size(), equalTo(2));
|
||||
assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1"));
|
||||
assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2"));
|
||||
}
|
||||
|
||||
@Test public void testCustomScriptBoost() throws Exception {
|
||||
// execute a search before we create an index
|
||||
try {
|
||||
client.prepareSearch().setQuery(termQuery("test", "value")).execute().actionGet();
|
||||
assert false : "should fail";
|
||||
} catch (Exception e) {
|
||||
// ignore, no indices
|
||||
}
|
||||
|
||||
try {
|
||||
client.prepareSearch("test").setQuery(termQuery("test", "value")).execute().actionGet();
|
||||
assert false : "should fail";
|
||||
} catch (Exception e) {
|
||||
// ignore, no indices
|
||||
}
|
||||
|
||||
client.admin().indices().create(createIndexRequest("test")).actionGet();
|
||||
client.index(indexRequest("test").type("type1").id("1")
|
||||
.source(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject())).actionGet();
|
||||
client.index(indexRequest("test").type("type1").id("2")
|
||||
.source(jsonBuilder().startObject().field("test", "value check").field("num1", 2.0f).endObject())).actionGet();
|
||||
client.admin().indices().refresh(refreshRequest()).actionGet();
|
||||
|
||||
logger.info("--- QUERY_THEN_FETCH");
|
||||
|
||||
logger.info("running doc['num1'].value");
|
||||
SearchResponse response = client.search(searchRequest()
|
||||
.searchType(SearchType.QUERY_THEN_FETCH)
|
||||
.source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("doc['num1'].value").lang("python")))
|
||||
).actionGet();
|
||||
|
||||
assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
|
||||
|
||||
assertThat(response.hits().totalHits(), equalTo(2l));
|
||||
logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation());
|
||||
logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation());
|
||||
assertThat(response.hits().getAt(0).id(), equalTo("2"));
|
||||
assertThat(response.hits().getAt(1).id(), equalTo("1"));
|
||||
|
||||
logger.info("running -doc['num1'].value");
|
||||
response = client.search(searchRequest()
|
||||
.searchType(SearchType.QUERY_THEN_FETCH)
|
||||
.source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("-doc['num1'].value").lang("python")))
|
||||
).actionGet();
|
||||
|
||||
assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
|
||||
|
||||
assertThat(response.hits().totalHits(), equalTo(2l));
|
||||
logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation());
|
||||
logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation());
|
||||
assertThat(response.hits().getAt(0).id(), equalTo("1"));
|
||||
assertThat(response.hits().getAt(1).id(), equalTo("2"));
|
||||
|
||||
|
||||
logger.info("running doc['num1'].value * score");
|
||||
response = client.search(searchRequest()
|
||||
.searchType(SearchType.QUERY_THEN_FETCH)
|
||||
.source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("doc['num1'].value * score").lang("python")))
|
||||
).actionGet();
|
||||
|
||||
assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
|
||||
|
||||
assertThat(response.hits().totalHits(), equalTo(2l));
|
||||
logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation());
|
||||
logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation());
|
||||
assertThat(response.hits().getAt(0).id(), equalTo("2"));
|
||||
assertThat(response.hits().getAt(1).id(), equalTo("1"));
|
||||
|
||||
logger.info("running param1 * param2 * score");
|
||||
response = client.search(searchRequest()
|
||||
.searchType(SearchType.QUERY_THEN_FETCH)
|
||||
.source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("param1 * param2 * score").param("param1", 2).param("param2", 2).lang("python")))
|
||||
).actionGet();
|
||||
|
||||
assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
|
||||
|
||||
assertThat(response.hits().totalHits(), equalTo(2l));
|
||||
logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation());
|
||||
logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
/*
|
||||
* Licensed to Elastic Search and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. Elastic Search licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.script.python;
|
||||
|
||||
import org.elasticsearch.common.StopWatch;
|
||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.script.ExecutableScript;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* @author kimchy (shay.banon)
|
||||
*/
|
||||
public class SimpleBench {
|
||||
|
||||
public static void main(String[] args) {
|
||||
PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS);
|
||||
Object compiled = se.compile("x + y");
|
||||
|
||||
Map<String, Object> vars = new HashMap<String, Object>();
|
||||
// warm up
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
vars.put("x", i);
|
||||
vars.put("y", i + 1);
|
||||
se.execute(compiled, vars);
|
||||
}
|
||||
|
||||
final long ITER = 100000;
|
||||
|
||||
StopWatch stopWatch = new StopWatch().start();
|
||||
for (long i = 0; i < ITER; i++) {
|
||||
se.execute(compiled, vars);
|
||||
}
|
||||
System.out.println("Execute Took: " + stopWatch.stop().lastTaskTime());
|
||||
|
||||
stopWatch = new StopWatch().start();
|
||||
ExecutableScript executableScript = se.executable(compiled, vars);
|
||||
for (long i = 0; i < ITER; i++) {
|
||||
executableScript.run();
|
||||
}
|
||||
System.out.println("Executable Took: " + stopWatch.stop().lastTaskTime());
|
||||
|
||||
stopWatch = new StopWatch().start();
|
||||
executableScript = se.executable(compiled, vars);
|
||||
for (long i = 0; i < ITER; i++) {
|
||||
executableScript.run(vars);
|
||||
}
|
||||
System.out.println("Executable (vars) Took: " + stopWatch.stop().lastTaskTime());
|
||||
}
|
||||
}
|
||||
|
|
@ -15,6 +15,7 @@ include 'plugins-mapper-attachments'
|
|||
|
||||
include 'plugins-lang-groovy'
|
||||
include 'plugins-lang-javascript'
|
||||
include 'plugins-lang-python'
|
||||
//include 'plugins-lang-ruby' #disabled for now ...
|
||||
|
||||
include 'plugins-transport-memcached'
|
||||
|
|
|
|||
Loading…
Reference in New Issue