From 8b5223fcfd644cd854a2a3dbdada2af7b7e92d10 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 5 Dec 2011 17:54:04 +0200 Subject: [PATCH 01/73] first commit --- .gitignore | 7 + README.md | 15 ++ pom.xml | 129 +++++++++ src/main/assemblies/plugin.xml | 26 ++ .../plugin/python/PythonPlugin.java | 48 ++++ .../python/PythonScriptEngineService.java | 229 ++++++++++++++++ src/main/resources/es-plugin.properties | 1 + .../python/PythonScriptEngineTests.java | 150 +++++++++++ .../python/PythonScriptMultiThreadedTest.java | 173 ++++++++++++ .../python/PythonScriptSearchTests.java | 253 ++++++++++++++++++ .../script/python/SimpleBench.java | 71 +++++ src/test/resources/log4j.properties | 5 + 12 files changed, 1107 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 pom.xml create mode 100644 src/main/assemblies/plugin.xml create mode 100644 src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java create mode 100644 src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java create mode 100644 src/main/resources/es-plugin.properties create mode 100644 src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java create mode 100644 src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java create mode 100644 src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java create mode 100644 src/test/java/org/elasticsearch/script/python/SimpleBench.java create mode 100644 src/test/resources/log4j.properties diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000000..06a1e6fedb6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +/data +/work +/logs +/.idea +/target +.DS_Store +*.iml diff --git a/README.md b/README.md new file mode 100644 index 00000000000..481b12f486a --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +Python lang Plugin for ElasticSearch +================================== + +The Python (jython) language plugin allows to have `python` as the language of scripts to execute. + +In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/1.0.0`. + + --------------------------------------- + | Python Plugin | ElasticSearch | + --------------------------------------- + | master | 0.18 -> master | + --------------------------------------- + | 1.0.0 | 0.18 -> master | + --------------------------------------- + diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000000..471acb47c64 --- /dev/null +++ b/pom.xml @@ -0,0 +1,129 @@ + + + elasticsearch-lang-python + 4.0.0 + org.elasticsearch + elasticsearch-lang-python + 1.0.0 + jar + JavaScript lang plugin for ElasticSearch + 2009 + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + scm:git:git@github.com:elasticsearch/elasticsearch-lang-python.git + scm:git:git@github.com:elasticsearch/elasticsearch-lang-python.git + http://github.com/elasticsearch/elasticsearch-lang-python + + + + org.sonatype.oss + oss-parent + 7 + + + + 0.18.5 + + + + + + + + org.elasticsearch + elasticsearch + ${elasticsearch.version} + compile + + + + org.python + jython-standalone + 2.5.2 + compile + + + + + + log4j + log4j + 1.2.16 + runtime + + + + org.testng + testng + 6.3.1 + test + + + + org.hamcrest + hamcrest-core + 1.3.RC2 + test + + + + org.hamcrest + hamcrest-library + 1.3.RC2 + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.3.2 + + 1.6 + 1.6 + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.11 + + + **/*Tests.java + + + + + org.apache.maven.plugins + maven-source-plugin + 2.1.2 + + + attach-sources + + jar + + + + + + maven-assembly-plugin + + + ${basedir}/src/main/assemblies/plugin.xml + + + + + + \ No newline at end of file diff --git a/src/main/assemblies/plugin.xml b/src/main/assemblies/plugin.xml new file mode 100644 index 00000000000..d5e43a7281f --- /dev/null +++ b/src/main/assemblies/plugin.xml @@ -0,0 +1,26 @@ + + + + + zip + + false + + + / + true + true + + org.elasticsearch:elasticsearch + + + + / + true + true + + org.python:jython-standalone + + + + \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java b/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java new file mode 100644 index 00000000000..06ff7d7f784 --- /dev/null +++ b/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java @@ -0,0 +1,48 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugin.python; + +import org.elasticsearch.common.inject.Module; +import org.elasticsearch.plugins.AbstractPlugin; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.python.PythonScriptEngineService; + +/** + * + */ +public class PythonPlugin extends AbstractPlugin { + + @Override + public String name() { + return "lang-python"; + } + + @Override + public String description() { + return "Python plugin allowing to add javascript scripting support"; + } + + @Override + public void processModule(Module module) { + if (module instanceof ScriptModule) { + ((ScriptModule) module).addScriptEngine(PythonScriptEngineService.class); + } + } +} diff --git a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java new file mode 100644 index 00000000000..84a00b3aa04 --- /dev/null +++ b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java @@ -0,0 +1,229 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.python; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.Scorer; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.ScriptEngineService; +import org.elasticsearch.script.SearchScript; +import org.elasticsearch.search.lookup.SearchLookup; +import org.python.core.Py; +import org.python.core.PyCode; +import org.python.core.PyObject; +import org.python.core.PyStringMap; +import org.python.util.PythonInterpreter; + +import java.util.Map; + +/** + * + */ +//TODO we can optimize the case for Map similar to PyStringMap +public class PythonScriptEngineService extends AbstractComponent implements ScriptEngineService { + + private final PythonInterpreter interp; + + @Inject + public PythonScriptEngineService(Settings settings) { + super(settings); + + this.interp = PythonInterpreter.threadLocalStateInterpreter(null); + } + + @Override + public String[] types() { + return new String[]{"python", "py"}; + } + + @Override + public String[] extensions() { + return new String[]{"py"}; + } + + @Override + public Object compile(String script) { + return interp.compile(script); + } + + @Override + public ExecutableScript executable(Object compiledScript, Map vars) { + return new PythonExecutableScript((PyCode) compiledScript, vars); + } + + @Override + public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable Map vars) { + return new PythonSearchScript((PyCode) compiledScript, vars, lookup); + } + + @Override + public Object execute(Object compiledScript, Map vars) { + PyObject pyVars = Py.java2py(vars); + interp.setLocals(pyVars); + PyObject ret = interp.eval((PyCode) compiledScript); + if (ret == null) { + return null; + } + return ret.__tojava__(Object.class); + } + + @Override + public Object unwrap(Object value) { + return unwrapValue(value); + } + + @Override + public void close() { + interp.cleanup(); + } + + public class PythonExecutableScript implements ExecutableScript { + + private final PyCode code; + + private final PyStringMap pyVars; + + public PythonExecutableScript(PyCode code, Map vars) { + this.code = code; + this.pyVars = new PyStringMap(); + for (Map.Entry entry : vars.entrySet()) { + pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue())); + } + } + + @Override + public void setNextVar(String name, Object value) { + pyVars.__setitem__(name, Py.java2py(value)); + } + + @Override + public Object run() { + interp.setLocals(pyVars); + PyObject ret = interp.eval(code); + if (ret == null) { + return null; + } + return ret.__tojava__(Object.class); + } + + @Override + public Object unwrap(Object value) { + return unwrapValue(value); + } + } + + public class PythonSearchScript implements SearchScript { + + private final PyCode code; + + private final PyStringMap pyVars; + + private final SearchLookup lookup; + + public PythonSearchScript(PyCode code, Map vars, SearchLookup lookup) { + this.code = code; + this.pyVars = new PyStringMap(); + for (Map.Entry entry : lookup.asMap().entrySet()) { + pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue())); + } + if (vars != null) { + for (Map.Entry entry : vars.entrySet()) { + pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue())); + } + } + this.lookup = lookup; + } + + @Override + public void setScorer(Scorer scorer) { + lookup.setScorer(scorer); + } + + @Override + public void setNextReader(IndexReader reader) { + lookup.setNextReader(reader); + } + + @Override + public void setNextDocId(int doc) { + lookup.setNextDocId(doc); + } + + @Override + public void setNextSource(Map source) { + lookup.source().setNextSource(source); + } + + @Override + public void setNextScore(float score) { + pyVars.__setitem__("_score", Py.java2py(score)); + } + + @Override + public void setNextVar(String name, Object value) { + pyVars.__setitem__(name, Py.java2py(value)); + } + + @Override + public Object run() { + interp.setLocals(pyVars); + PyObject ret = interp.eval(code); + if (ret == null) { + return null; + } + return ret.__tojava__(Object.class); + } + + @Override + public float runAsFloat() { + return ((Number) run()).floatValue(); + } + + @Override + public long runAsLong() { + return ((Number) run()).longValue(); + } + + @Override + public double runAsDouble() { + return ((Number) run()).doubleValue(); + } + + @Override + public Object unwrap(Object value) { + return unwrapValue(value); + } + } + + + public static Object unwrapValue(Object value) { + if (value == null) { + return null; + } else if (value instanceof PyObject) { + // seems like this is enough, inner PyDictionary will do the conversion for us for example, so expose it directly + return ((PyObject) value).__tojava__(Object.class); + } + return value; + } +} diff --git a/src/main/resources/es-plugin.properties b/src/main/resources/es-plugin.properties new file mode 100644 index 00000000000..a24df54c363 --- /dev/null +++ b/src/main/resources/es-plugin.properties @@ -0,0 +1 @@ +plugin=org.elasticsearch.plugin.python.PythonPlugin diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java new file mode 100644 index 00000000000..f2db686efe8 --- /dev/null +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java @@ -0,0 +1,150 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.python; + +import org.elasticsearch.common.collect.Lists; +import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.script.ExecutableScript; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +/** + * + */ +@Test +public class PythonScriptEngineTests { + + private PythonScriptEngineService se; + + @BeforeClass + public void setup() { + se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS); + } + + @AfterClass + public void close() { + se.close(); + } + + @Test + public void testSimpleEquation() { + Map vars = new HashMap(); + Object o = se.execute(se.compile("1 + 2"), vars); + assertThat(((Number) o).intValue(), equalTo(3)); + } + + @Test + public void testMapAccess() { + Map vars = new HashMap(); + + Map obj2 = MapBuilder.newMapBuilder().put("prop2", "value2").map(); + Map obj1 = MapBuilder.newMapBuilder().put("prop1", "value1").put("obj2", obj2).put("l", Lists.newArrayList("2", "1")).map(); + vars.put("obj1", obj1); + Object o = se.execute(se.compile("obj1"), vars); + assertThat(o, instanceOf(Map.class)); + obj1 = (Map) o; + assertThat((String) obj1.get("prop1"), equalTo("value1")); + assertThat((String) ((Map) obj1.get("obj2")).get("prop2"), equalTo("value2")); + + o = se.execute(se.compile("obj1['l'][0]"), vars); + assertThat(((String) o), equalTo("2")); + } + + @Test + public void testObjectMapInter() { + Map vars = new HashMap(); + Map ctx = new HashMap(); + Map obj1 = new HashMap(); + obj1.put("prop1", "value1"); + ctx.put("obj1", obj1); + vars.put("ctx", ctx); + + se.execute(se.compile("ctx['obj2'] = { 'prop2' : 'value2' }; ctx['obj1']['prop1'] = 'uvalue1'"), vars); + ctx = (Map) se.unwrap(vars.get("ctx")); + assertThat(ctx.containsKey("obj1"), equalTo(true)); + assertThat((String) ((Map) ctx.get("obj1")).get("prop1"), equalTo("uvalue1")); + assertThat(ctx.containsKey("obj2"), equalTo(true)); + assertThat((String) ((Map) ctx.get("obj2")).get("prop2"), equalTo("value2")); + } + + @Test + public void testAccessListInScript() { + Map vars = new HashMap(); + Map obj2 = MapBuilder.newMapBuilder().put("prop2", "value2").map(); + Map obj1 = MapBuilder.newMapBuilder().put("prop1", "value1").put("obj2", obj2).map(); + vars.put("l", Lists.newArrayList("1", "2", "3", obj1)); + +// Object o = se.execute(se.compile("l.length"), vars); +// assertThat(((Number) o).intValue(), equalTo(4)); + + Object o = se.execute(se.compile("l[0]"), vars); + assertThat(((String) o), equalTo("1")); + + o = se.execute(se.compile("l[3]"), vars); + obj1 = (Map) o; + assertThat((String) obj1.get("prop1"), equalTo("value1")); + assertThat((String) ((Map) obj1.get("obj2")).get("prop2"), equalTo("value2")); + + o = se.execute(se.compile("l[3]['prop1']"), vars); + assertThat(((String) o), equalTo("value1")); + } + + @Test + public void testChangingVarsCrossExecution1() { + Map vars = new HashMap(); + Map ctx = new HashMap(); + vars.put("ctx", ctx); + Object compiledScript = se.compile("ctx['value']"); + + ExecutableScript script = se.executable(compiledScript, vars); + ctx.put("value", 1); + Object o = script.run(); + assertThat(((Number) o).intValue(), equalTo(1)); + + ctx.put("value", 2); + o = script.run(); + assertThat(((Number) o).intValue(), equalTo(2)); + } + + @Test + public void testChangingVarsCrossExecution2() { + Map vars = new HashMap(); + Map ctx = new HashMap(); + Object compiledScript = se.compile("value"); + + ExecutableScript script = se.executable(compiledScript, vars); + script.setNextVar("value", 1); + Object o = script.run(); + assertThat(((Number) o).intValue(), equalTo(1)); + + script.setNextVar("value", 2); + o = script.run(); + assertThat(((Number) o).intValue(), equalTo(2)); + } +} diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java b/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java new file mode 100644 index 00000000000..01488a43708 --- /dev/null +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java @@ -0,0 +1,173 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.python; + +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.util.concurrent.jsr166y.ThreadLocalRandom; +import org.elasticsearch.script.ExecutableScript; +import org.testng.annotations.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +/** + * + */ +@Test +public class PythonScriptMultiThreadedTest { + + protected final ESLogger logger = Loggers.getLogger(getClass()); + + @Test + public void testExecutableNoRuntimeParams() throws Exception { + final PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS); + final Object compiled = se.compile("x + y"); + final AtomicBoolean failed = new AtomicBoolean(); + + Thread[] threads = new Thread[50]; + final CountDownLatch latch = new CountDownLatch(threads.length); + final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1); + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(new Runnable() { + @Override + public void run() { + try { + barrier.await(); + long x = ThreadLocalRandom.current().nextInt(); + long y = ThreadLocalRandom.current().nextInt(); + long addition = x + y; + Map vars = new HashMap(); + vars.put("x", x); + vars.put("y", y); + ExecutableScript script = se.executable(compiled, vars); + for (int i = 0; i < 100000; i++) { + long result = ((Number) script.run()).longValue(); + assertThat(result, equalTo(addition)); + } + } catch (Throwable t) { + failed.set(true); + logger.error("failed", t); + } finally { + latch.countDown(); + } + } + }); + } + for (int i = 0; i < threads.length; i++) { + threads[i].start(); + } + barrier.await(); + latch.await(); + assertThat(failed.get(), equalTo(false)); + } + + +// @Test public void testExecutableWithRuntimeParams() throws Exception { +// final PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS); +// final Object compiled = se.compile("x + y"); +// final AtomicBoolean failed = new AtomicBoolean(); +// +// Thread[] threads = new Thread[50]; +// final CountDownLatch latch = new CountDownLatch(threads.length); +// final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1); +// for (int i = 0; i < threads.length; i++) { +// threads[i] = new Thread(new Runnable() { +// @Override public void run() { +// try { +// barrier.await(); +// long x = ThreadLocalRandom.current().nextInt(); +// Map vars = new HashMap(); +// vars.put("x", x); +// ExecutableScript script = se.executable(compiled, vars); +// Map runtimeVars = new HashMap(); +// for (int i = 0; i < 100000; i++) { +// long y = ThreadLocalRandom.current().nextInt(); +// long addition = x + y; +// runtimeVars.put("y", y); +// long result = ((Number) script.run(runtimeVars)).longValue(); +// assertThat(result, equalTo(addition)); +// } +// } catch (Throwable t) { +// failed.set(true); +// logger.error("failed", t); +// } finally { +// latch.countDown(); +// } +// } +// }); +// } +// for (int i = 0; i < threads.length; i++) { +// threads[i].start(); +// } +// barrier.await(); +// latch.await(); +// assertThat(failed.get(), equalTo(false)); +// } + + @Test + public void testExecute() throws Exception { + final PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS); + final Object compiled = se.compile("x + y"); + final AtomicBoolean failed = new AtomicBoolean(); + + Thread[] threads = new Thread[50]; + final CountDownLatch latch = new CountDownLatch(threads.length); + final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1); + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(new Runnable() { + @Override + public void run() { + try { + barrier.await(); + Map runtimeVars = new HashMap(); + for (int i = 0; i < 100000; i++) { + long x = ThreadLocalRandom.current().nextInt(); + long y = ThreadLocalRandom.current().nextInt(); + long addition = x + y; + runtimeVars.put("x", x); + runtimeVars.put("y", y); + long result = ((Number) se.execute(compiled, runtimeVars)).longValue(); + assertThat(result, equalTo(addition)); + } + } catch (Throwable t) { + failed.set(true); + logger.error("failed", t); + } finally { + latch.countDown(); + } + } + }); + } + for (int i = 0; i < threads.length; i++) { + threads[i].start(); + } + barrier.await(); + latch.await(); + assertThat(failed.get(), equalTo(false)); + } +} diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java new file mode 100644 index 00000000000..0f5ef643dfc --- /dev/null +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java @@ -0,0 +1,253 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.python; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.network.NetworkUtils; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeBuilder; +import org.elasticsearch.search.sort.SortOrder; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.client.Requests.*; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.FilterBuilders.scriptFilter; +import static org.elasticsearch.index.query.QueryBuilders.*; +import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +/** + * + */ +@Test +public class PythonScriptSearchTests { + + protected final ESLogger logger = Loggers.getLogger(getClass()); + + private Node node; + + private Client client; + + @BeforeMethod + public void createNodes() throws Exception { + node = NodeBuilder.nodeBuilder().settings(ImmutableSettings.settingsBuilder() + .put("path.data", "target/data") + .put("cluster.name", "test-cluster-" + NetworkUtils.getLocalAddress()) + .put("gateway.type", "none") + .put("number_of_shards", 1)).node(); + client = node.client(); + } + + @AfterMethod + public void closeNodes() { + client.close(); + node.close(); + } + + @Test + public void testPythonFilter() throws Exception { + client.admin().indices().prepareCreate("test").execute().actionGet(); + client.prepareIndex("test", "type1", "1") + .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject()) + .execute().actionGet(); + client.admin().indices().prepareFlush().execute().actionGet(); + client.prepareIndex("test", "type1", "2") + .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject()) + .execute().actionGet(); + client.admin().indices().prepareFlush().execute().actionGet(); + client.prepareIndex("test", "type1", "3") + .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).endObject()) + .execute().actionGet(); + client.admin().indices().refresh(refreshRequest()).actionGet(); + + logger.info("running doc['num1'].value > 1"); + SearchResponse response = client.prepareSearch() + .setQuery(filteredQuery(matchAllQuery(), scriptFilter("doc['num1'].value > 1").lang("python"))) + .addSort("num1", SortOrder.ASC) + .addScriptField("sNum1", "python", "doc['num1'].value", null) + .execute().actionGet(); + + assertThat(response.hits().totalHits(), equalTo(2l)); + assertThat(response.hits().getAt(0).id(), equalTo("2")); + assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0)); + assertThat(response.hits().getAt(1).id(), equalTo("3")); + assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(3.0)); + + logger.info("running doc['num1'].value > param1"); + response = client.prepareSearch() + .setQuery(filteredQuery(matchAllQuery(), scriptFilter("doc['num1'].value > param1").lang("python").addParam("param1", 2))) + .addSort("num1", SortOrder.ASC) + .addScriptField("sNum1", "python", "doc['num1'].value", null) + .execute().actionGet(); + + assertThat(response.hits().totalHits(), equalTo(1l)); + assertThat(response.hits().getAt(0).id(), equalTo("3")); + assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(3.0)); + + logger.info("running doc['num1'].value > param1"); + response = client.prepareSearch() + .setQuery(filteredQuery(matchAllQuery(), scriptFilter("doc['num1'].value > param1").lang("python").addParam("param1", -1))) + .addSort("num1", SortOrder.ASC) + .addScriptField("sNum1", "python", "doc['num1'].value", null) + .execute().actionGet(); + + assertThat(response.hits().totalHits(), equalTo(3l)); + assertThat(response.hits().getAt(0).id(), equalTo("1")); + assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0)); + assertThat(response.hits().getAt(1).id(), equalTo("2")); + assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0)); + assertThat(response.hits().getAt(2).id(), equalTo("3")); + assertThat((Double) response.hits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0)); + } + + @Test + public void testScriptFieldUsingSource() throws Exception { + client.admin().indices().prepareCreate("test").execute().actionGet(); + client.prepareIndex("test", "type1", "1") + .setSource(jsonBuilder().startObject() + .startObject("obj1").field("test", "something").endObject() + .startObject("obj2").startArray("arr2").value("arr_value1").value("arr_value2").endArray().endObject() + .endObject()) + .execute().actionGet(); + client.admin().indices().refresh(refreshRequest()).actionGet(); + + SearchResponse response = client.prepareSearch() + .setQuery(matchAllQuery()) + .addField("_source.obj1") // we also automatically detect _source in fields + .addScriptField("s_obj1", "python", "_source['obj1']", null) + .addScriptField("s_obj1_test", "python", "_source['obj1']['test']", null) + .addScriptField("s_obj2", "python", "_source['obj2']", null) + .addScriptField("s_obj2_arr2", "python", "_source['obj2']['arr2']", null) + .execute().actionGet(); + + Map sObj1 = (Map) response.hits().getAt(0).field("_source.obj1").value(); + assertThat(sObj1.get("test").toString(), equalTo("something")); + assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something")); + + sObj1 = (Map) response.hits().getAt(0).field("s_obj1").value(); + assertThat(sObj1.get("test").toString(), equalTo("something")); + assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something")); + + Map sObj2 = (Map) response.hits().getAt(0).field("s_obj2").value(); + List sObj2Arr2 = (List) sObj2.get("arr2"); + assertThat(sObj2Arr2.size(), equalTo(2)); + assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1")); + assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2")); + + sObj2Arr2 = (List) response.hits().getAt(0).field("s_obj2_arr2").value(); + assertThat(sObj2Arr2.size(), equalTo(2)); + assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1")); + assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2")); + } + + @Test + public void testCustomScriptBoost() throws Exception { + // execute a search before we create an index + try { + client.prepareSearch().setQuery(termQuery("test", "value")).execute().actionGet(); + assert false : "should fail"; + } catch (Exception e) { + // ignore, no indices + } + + try { + client.prepareSearch("test").setQuery(termQuery("test", "value")).execute().actionGet(); + assert false : "should fail"; + } catch (Exception e) { + // ignore, no indices + } + + client.admin().indices().create(createIndexRequest("test")).actionGet(); + client.index(indexRequest("test").type("type1").id("1") + .source(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject())).actionGet(); + client.index(indexRequest("test").type("type1").id("2") + .source(jsonBuilder().startObject().field("test", "value check").field("num1", 2.0f).endObject())).actionGet(); + client.admin().indices().refresh(refreshRequest()).actionGet(); + + logger.info("--- QUERY_THEN_FETCH"); + + logger.info("running doc['num1'].value"); + SearchResponse response = client.search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("doc['num1'].value").lang("python"))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0)); + + assertThat(response.hits().totalHits(), equalTo(2l)); + logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation()); + logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation()); + assertThat(response.hits().getAt(0).id(), equalTo("2")); + assertThat(response.hits().getAt(1).id(), equalTo("1")); + + logger.info("running -doc['num1'].value"); + response = client.search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("-doc['num1'].value").lang("python"))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0)); + + assertThat(response.hits().totalHits(), equalTo(2l)); + logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation()); + logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation()); + assertThat(response.hits().getAt(0).id(), equalTo("1")); + assertThat(response.hits().getAt(1).id(), equalTo("2")); + + + logger.info("running doc['num1'].value * _score"); + response = client.search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("doc['num1'].value * _score").lang("python"))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0)); + + assertThat(response.hits().totalHits(), equalTo(2l)); + logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation()); + logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation()); + assertThat(response.hits().getAt(0).id(), equalTo("2")); + assertThat(response.hits().getAt(1).id(), equalTo("1")); + + logger.info("running param1 * param2 * _score"); + response = client.search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("param1 * param2 * _score").param("param1", 2).param("param2", 2).lang("python"))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0)); + + assertThat(response.hits().totalHits(), equalTo(2l)); + logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation()); + logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation()); + } +} diff --git a/src/test/java/org/elasticsearch/script/python/SimpleBench.java b/src/test/java/org/elasticsearch/script/python/SimpleBench.java new file mode 100644 index 00000000000..362e838a9f0 --- /dev/null +++ b/src/test/java/org/elasticsearch/script/python/SimpleBench.java @@ -0,0 +1,71 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.python; + +import org.elasticsearch.common.StopWatch; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.script.ExecutableScript; + +import java.util.HashMap; +import java.util.Map; + +/** + * + */ +public class SimpleBench { + + public static void main(String[] args) { + PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS); + Object compiled = se.compile("x + y"); + + Map vars = new HashMap(); + // warm up + for (int i = 0; i < 1000; i++) { + vars.put("x", i); + vars.put("y", i + 1); + se.execute(compiled, vars); + } + + final long ITER = 100000; + + StopWatch stopWatch = new StopWatch().start(); + for (long i = 0; i < ITER; i++) { + se.execute(compiled, vars); + } + System.out.println("Execute Took: " + stopWatch.stop().lastTaskTime()); + + stopWatch = new StopWatch().start(); + ExecutableScript executableScript = se.executable(compiled, vars); + for (long i = 0; i < ITER; i++) { + executableScript.run(); + } + System.out.println("Executable Took: " + stopWatch.stop().lastTaskTime()); + + stopWatch = new StopWatch().start(); + executableScript = se.executable(compiled, vars); + for (long i = 0; i < ITER; i++) { + for (Map.Entry entry : vars.entrySet()) { + executableScript.setNextVar(entry.getKey(), entry.getValue()); + } + executableScript.run(); + } + System.out.println("Executable (vars) Took: " + stopWatch.stop().lastTaskTime()); + } +} diff --git a/src/test/resources/log4j.properties b/src/test/resources/log4j.properties new file mode 100644 index 00000000000..497c97f9959 --- /dev/null +++ b/src/test/resources/log4j.properties @@ -0,0 +1,5 @@ +log4j.rootLogger=INFO, out + +log4j.appender.out=org.apache.log4j.ConsoleAppender +log4j.appender.out.layout=org.apache.log4j.PatternLayout +log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n From 17d1550f6e3676d5fd632e0fe97034c2e91b1026 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 5 Dec 2011 17:59:06 +0200 Subject: [PATCH 02/73] move to 1.1.0 snap --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 471acb47c64..d4df457ccd1 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 4.0.0 org.elasticsearch elasticsearch-lang-python - 1.0.0 + 1.1.0-SNAPSHOT jar JavaScript lang plugin for ElasticSearch 2009 From 65ada1b75a1bfa1a27f2d3be87e0194cec809fc9 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 13 Dec 2011 15:06:16 +0200 Subject: [PATCH 03/73] fix pom --- pom.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pom.xml b/pom.xml index d4df457ccd1..3e746c442f4 100644 --- a/pom.xml +++ b/pom.xml @@ -119,10 +119,19 @@ maven-assembly-plugin + ${project.build.directory}/releases/ ${basedir}/src/main/assemblies/plugin.xml + + + package + + single + + + From 1b85f22e0a0141a923835856f7c231f93464afaf Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 31 Jan 2012 13:13:11 +0200 Subject: [PATCH 04/73] move to elasticsearch 0.19.0 snap and use some of its features --- README.md | 2 +- pom.xml | 2 +- .../org/elasticsearch/plugin/python/PythonPlugin.java | 8 ++------ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 481b12f486a..4a9b4e054a3 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e --------------------------------------- | Python Plugin | ElasticSearch | --------------------------------------- - | master | 0.18 -> master | + | master | master (0.19) | --------------------------------------- | 1.0.0 | 0.18 -> master | --------------------------------------- diff --git a/pom.xml b/pom.xml index 3e746c442f4..64731c3d141 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ - 0.18.5 + 0.19.0-SNAPSHOT diff --git a/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java b/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java index 06ff7d7f784..6a54b8a1236 100644 --- a/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java +++ b/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java @@ -19,7 +19,6 @@ package org.elasticsearch.plugin.python; -import org.elasticsearch.common.inject.Module; import org.elasticsearch.plugins.AbstractPlugin; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.python.PythonScriptEngineService; @@ -39,10 +38,7 @@ public class PythonPlugin extends AbstractPlugin { return "Python plugin allowing to add javascript scripting support"; } - @Override - public void processModule(Module module) { - if (module instanceof ScriptModule) { - ((ScriptModule) module).addScriptEngine(PythonScriptEngineService.class); - } + public void onModule(ScriptModule module) { + module.addScriptEngine(PythonScriptEngineService.class); } } From a581da9ee268db1c3f69a11dd6642bcb54f9eeb6 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 7 Feb 2012 16:45:13 +0200 Subject: [PATCH 05/73] release 1.1.0 supporting elasticsearch 0.19 --- README.md | 8 +++++--- pom.xml | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4a9b4e054a3..28308ee3a8e 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,15 @@ Python lang Plugin for ElasticSearch The Python (jython) language plugin allows to have `python` as the language of scripts to execute. -In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/1.0.0`. +In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/1.1.0`. --------------------------------------- | Python Plugin | ElasticSearch | --------------------------------------- - | master | master (0.19) | + | master | 0.19 -> master | --------------------------------------- - | 1.0.0 | 0.18 -> master | + | 1.1.0 | 0.19 -> master | + --------------------------------------- + | 1.0.0 | 0.18 | --------------------------------------- diff --git a/pom.xml b/pom.xml index 64731c3d141..c1648242586 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 4.0.0 org.elasticsearch elasticsearch-lang-python - 1.1.0-SNAPSHOT + 1.1.0 jar JavaScript lang plugin for ElasticSearch 2009 @@ -30,7 +30,7 @@ - 0.19.0-SNAPSHOT + 0.19.0.RC1 From f62fab14da8124d34b6b18ce4b654c9544db1f38 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 7 Feb 2012 16:45:38 +0200 Subject: [PATCH 06/73] move to 1.2.0 snap --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c1648242586..f59012c54ea 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 4.0.0 org.elasticsearch elasticsearch-lang-python - 1.1.0 + 1.2.0-SNAPSHOT jar JavaScript lang plugin for ElasticSearch 2009 From dc80e6532ecb1262beea6813c44b800a51dccff3 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 26 Feb 2012 10:20:53 +0200 Subject: [PATCH 07/73] latest elasticsearch --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f59012c54ea..6ad26f04129 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ - 0.19.0.RC1 + 0.19.0.RC3 From 2023a1000216b9667966ac838f13f94435329452 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 26 Feb 2012 10:21:19 +0200 Subject: [PATCH 08/73] latest assembly --- pom.xml | 2 ++ src/main/assemblies/plugin.xml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6ad26f04129..d3405ae6543 100644 --- a/pom.xml +++ b/pom.xml @@ -118,7 +118,9 @@ maven-assembly-plugin + 2.2.2 + false ${project.build.directory}/releases/ ${basedir}/src/main/assemblies/plugin.xml diff --git a/src/main/assemblies/plugin.xml b/src/main/assemblies/plugin.xml index d5e43a7281f..037ea9f7ee8 100644 --- a/src/main/assemblies/plugin.xml +++ b/src/main/assemblies/plugin.xml @@ -1,6 +1,6 @@ - + plugin zip From 97d41c98344cde7253cd54c3eb2889ec5dc2423c Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 10 Jun 2012 22:12:36 +0200 Subject: [PATCH 09/73] add repo and license --- LICENSE.txt | 202 ++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 18 +++++ pom.xml | 6 +- 3 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 28308ee3a8e..7050f1c698a 100644 --- a/README.md +++ b/README.md @@ -15,3 +15,21 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e | 1.0.0 | 0.18 | --------------------------------------- +License +------- + + This software is licensed under the Apache 2 license, quoted below. + + Copyright 2009-2012 Shay Banon and ElasticSearch + + Licensed under the Apache License, Version 2.0 (the "License"); you may not + use this file except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations under + the License. diff --git a/pom.xml b/pom.xml index d3405ae6543..267f6aa1a99 100644 --- a/pom.xml +++ b/pom.xml @@ -30,10 +30,14 @@ - 0.19.0.RC3 + 0.19.0 + + sonatype + http://oss.sonatype.org/content/repositories/releases/ + From 935e512c3186484285ec9a49edd76c05c22ff5e6 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Sun, 24 Feb 2013 22:53:09 +0100 Subject: [PATCH 10/73] Move to Elasticsearch 0.21.0.Beta1 Due to refactoring in 0.21.x we have to update this plugin Closes #3. --- README.md | 4 +- pom.xml | 4 +- .../python/PythonScriptEngineService.java | 6 +- .../python/PythonScriptSearchTests.java | 86 +++++++++---------- 4 files changed, 50 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 7050f1c698a..46296fa4850 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e --------------------------------------- | Python Plugin | ElasticSearch | --------------------------------------- - | master | 0.19 -> master | + | master | 0.21 -> master | --------------------------------------- - | 1.1.0 | 0.19 -> master | + | 1.1.0 | 0.19 -> 0.20 | --------------------------------------- | 1.0.0 | 0.18 | --------------------------------------- diff --git a/pom.xml b/pom.xml index 267f6aa1a99..32e856afe9b 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ - 0.19.0 + 0.21.0.Beta1-SNAPSHOT @@ -141,4 +141,4 @@ - \ No newline at end of file + diff --git a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java index 84a00b3aa04..877c6f50c2a 100644 --- a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java @@ -19,7 +19,7 @@ package org.elasticsearch.script.python; -import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.search.Scorer; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; @@ -161,8 +161,8 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri } @Override - public void setNextReader(IndexReader reader) { - lookup.setNextReader(reader); + public void setNextReader(AtomicReaderContext context) { + lookup.setNextReader(context); } @Override diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java index 0f5ef643dfc..91038aa94b3 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java @@ -96,11 +96,11 @@ public class PythonScriptSearchTests { .addScriptField("sNum1", "python", "doc['num1'].value", null) .execute().actionGet(); - assertThat(response.hits().totalHits(), equalTo(2l)); - assertThat(response.hits().getAt(0).id(), equalTo("2")); - assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0)); - assertThat(response.hits().getAt(1).id(), equalTo("3")); - assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(3.0)); + assertThat(response.getHits().totalHits(), equalTo(2l)); + assertThat(response.getHits().getAt(0).id(), equalTo("2")); + assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0)); + assertThat(response.getHits().getAt(1).id(), equalTo("3")); + assertThat((Double) response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(3.0)); logger.info("running doc['num1'].value > param1"); response = client.prepareSearch() @@ -109,9 +109,9 @@ public class PythonScriptSearchTests { .addScriptField("sNum1", "python", "doc['num1'].value", null) .execute().actionGet(); - assertThat(response.hits().totalHits(), equalTo(1l)); - assertThat(response.hits().getAt(0).id(), equalTo("3")); - assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(3.0)); + assertThat(response.getHits().totalHits(), equalTo(1l)); + assertThat(response.getHits().getAt(0).id(), equalTo("3")); + assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(3.0)); logger.info("running doc['num1'].value > param1"); response = client.prepareSearch() @@ -120,13 +120,13 @@ public class PythonScriptSearchTests { .addScriptField("sNum1", "python", "doc['num1'].value", null) .execute().actionGet(); - assertThat(response.hits().totalHits(), equalTo(3l)); - assertThat(response.hits().getAt(0).id(), equalTo("1")); - assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0)); - assertThat(response.hits().getAt(1).id(), equalTo("2")); - assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0)); - assertThat(response.hits().getAt(2).id(), equalTo("3")); - assertThat((Double) response.hits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0)); + assertThat(response.getHits().totalHits(), equalTo(3l)); + assertThat(response.getHits().getAt(0).id(), equalTo("1")); + assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0)); + assertThat(response.getHits().getAt(1).id(), equalTo("2")); + assertThat((Double) response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0)); + assertThat(response.getHits().getAt(2).id(), equalTo("3")); + assertThat((Double) response.getHits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0)); } @Test @@ -149,21 +149,21 @@ public class PythonScriptSearchTests { .addScriptField("s_obj2_arr2", "python", "_source['obj2']['arr2']", null) .execute().actionGet(); - Map sObj1 = (Map) response.hits().getAt(0).field("_source.obj1").value(); + Map sObj1 = (Map) response.getHits().getAt(0).field("_source.obj1").value(); assertThat(sObj1.get("test").toString(), equalTo("something")); - assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something")); + assertThat(response.getHits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something")); - sObj1 = (Map) response.hits().getAt(0).field("s_obj1").value(); + sObj1 = (Map) response.getHits().getAt(0).field("s_obj1").value(); assertThat(sObj1.get("test").toString(), equalTo("something")); - assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something")); + assertThat(response.getHits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something")); - Map sObj2 = (Map) response.hits().getAt(0).field("s_obj2").value(); + Map sObj2 = (Map) response.getHits().getAt(0).field("s_obj2").value(); List sObj2Arr2 = (List) sObj2.get("arr2"); assertThat(sObj2Arr2.size(), equalTo(2)); assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1")); assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2")); - sObj2Arr2 = (List) response.hits().getAt(0).field("s_obj2_arr2").value(); + sObj2Arr2 = (List) response.getHits().getAt(0).field("s_obj2_arr2").value(); assertThat(sObj2Arr2.size(), equalTo(2)); assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1")); assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2")); @@ -201,13 +201,13 @@ public class PythonScriptSearchTests { .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("doc['num1'].value").lang("python"))) ).actionGet(); - assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0)); + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); - assertThat(response.hits().totalHits(), equalTo(2l)); - logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation()); - logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation()); - assertThat(response.hits().getAt(0).id(), equalTo("2")); - assertThat(response.hits().getAt(1).id(), equalTo("1")); + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info("Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info("Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + assertThat(response.getHits().getAt(0).id(), equalTo("2")); + assertThat(response.getHits().getAt(1).id(), equalTo("1")); logger.info("running -doc['num1'].value"); response = client.search(searchRequest() @@ -215,13 +215,13 @@ public class PythonScriptSearchTests { .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("-doc['num1'].value").lang("python"))) ).actionGet(); - assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0)); + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); - assertThat(response.hits().totalHits(), equalTo(2l)); - logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation()); - logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation()); - assertThat(response.hits().getAt(0).id(), equalTo("1")); - assertThat(response.hits().getAt(1).id(), equalTo("2")); + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info("Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info("Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + assertThat(response.getHits().getAt(0).id(), equalTo("1")); + assertThat(response.getHits().getAt(1).id(), equalTo("2")); logger.info("running doc['num1'].value * _score"); @@ -230,13 +230,13 @@ public class PythonScriptSearchTests { .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("doc['num1'].value * _score").lang("python"))) ).actionGet(); - assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0)); + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); - assertThat(response.hits().totalHits(), equalTo(2l)); - logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation()); - logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation()); - assertThat(response.hits().getAt(0).id(), equalTo("2")); - assertThat(response.hits().getAt(1).id(), equalTo("1")); + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info("Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info("Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + assertThat(response.getHits().getAt(0).id(), equalTo("2")); + assertThat(response.getHits().getAt(1).id(), equalTo("1")); logger.info("running param1 * param2 * _score"); response = client.search(searchRequest() @@ -244,10 +244,10 @@ public class PythonScriptSearchTests { .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("param1 * param2 * _score").param("param1", 2).param("param2", 2).lang("python"))) ).actionGet(); - assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0)); + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); - assertThat(response.hits().totalHits(), equalTo(2l)); - logger.info("Hit[0] {} Explanation {}", response.hits().getAt(0).id(), response.hits().getAt(0).explanation()); - logger.info("Hit[1] {} Explanation {}", response.hits().getAt(1).id(), response.hits().getAt(1).explanation()); + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info("Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info("Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); } } From 008532da151cbf34374a93e45378b28f1fb2a72c Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 26 Feb 2013 16:24:25 +0100 Subject: [PATCH 11/73] release 1.2.0 --- README.md | 6 ++++-- pom.xml | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 46296fa4850..d0769cc32fd 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,14 @@ Python lang Plugin for ElasticSearch The Python (jython) language plugin allows to have `python` as the language of scripts to execute. -In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/1.1.0`. +In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/1.2.0`. --------------------------------------- | Python Plugin | ElasticSearch | --------------------------------------- - | master | 0.21 -> master | + | master | 0.90 -> master | + --------------------------------------- + | 1.2.0 | 0.90 -> master | --------------------------------------- | 1.1.0 | 0.19 -> 0.20 | --------------------------------------- diff --git a/pom.xml b/pom.xml index 32e856afe9b..0011d49d3ad 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 4.0.0 org.elasticsearch elasticsearch-lang-python - 1.2.0-SNAPSHOT + 1.2.0 jar JavaScript lang plugin for ElasticSearch 2009 @@ -30,7 +30,7 @@ - 0.21.0.Beta1-SNAPSHOT + 0.90.0.Beta1 From 936b5952be2aaedf0d952bdde46a92840490f050 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 26 Feb 2013 16:24:41 +0100 Subject: [PATCH 12/73] move to 1.3 snap --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0011d49d3ad..f782217b230 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 4.0.0 org.elasticsearch elasticsearch-lang-python - 1.2.0 + 1.3.0-SNAPSHOT jar JavaScript lang plugin for ElasticSearch 2009 From ecbac6cea63150e34a10ab7c429d986fe8004b67 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 21 Aug 2013 11:46:13 +0200 Subject: [PATCH 13/73] Create CONTRIBUTING.md --- CONTRIBUTING.md | 98 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000000..0c59bbbc6ea --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,98 @@ +Contributing to elasticsearch +============================= + +Elasticsearch is an open source project and we love to receive contributions from our community — you! There are many ways to contribute, from writing tutorials or blog posts, improving the documentation, submitting bug reports and feature requests or writing code which can be incorporated into Elasticsearch itself. + +Bug reports +----------- + +If you think you have found a bug in Elasticsearch, first make sure that you are testing against the [latest version of Elasticsearch](http://www.elasticsearch.org/download/) - your issue may already have been fixed. If not, search our [issues list](https://github.com/elasticsearch/elasticsearch/issues) on GitHub in case a similar issue has already been opened. + +It is very helpful if you can prepare a reproduction of the bug. In other words, provide a small test case which we can run to confirm your bug. It makes it easier to find the problem and to fix it. Test cases should be provided as `curl` commands which we can copy and paste into a terminal to run it locally, for example: + +```sh +# delete the index +curl -XDELETE localhost:9200/test + +# insert a document +curl -XPUT localhost:9200/test/test/1 -d '{ + "title": "test document" +}' + +# this should return XXXX but instead returns YYY +curl .... +``` + +Provide as much information as you can. You may think that the problem lies with your query, when actually it depends on how your data is indexed. The easier it is for us to recreate your problem, the faster it is likely to be fixed. + +Feature requests +---------------- + +If you find yourself wishing for a feature that doesn't exist in Elasticsearch, you are probably not alone. There are bound to be others out there with similar needs. Many of the features that Elasticsearch has today have been added because our users saw the need. +Open an issue on our [issues list](https://github.com/elasticsearch/elasticsearch/issues) on GitHub which describes the feature you would like to see, why you need it, and how it should work. + +Contributing code and documentation changes +------------------------------------------- + +If you have a bugfix or new feature that you would like to contribute to Elasticsearch, please find or open an issue about it first. Talk about what you would like to do. It may be that somebody is already working on it, or that there are particular issues that you should know about before implementing the change. + +We enjoy working with contributors to get their code accepted. There are many approaches to fixing a problem and it is important to find the best approach before writing too much code. + +The process for contributing to any of the [Elasticsearch repositories](https://github.com/elasticsearch/) is similar. Details for individual projects can be found below. + +### Fork and clone the repository + +You will need to fork the main Elasticsearch code or documentation repository and clone it to your local machine. See +[github help page](https://help.github.com/articles/fork-a-repo) for help. + +Further instructions for specific projects are given below. + +### Submitting your changes + +Once your changes and tests are ready to submit for review: + +1. Test your changes +Run the test suite to make sure that nothing is broken. + +2. Sign the Contributor License Agreement +Please make sure you have signed our [Contributor License Agreement](http://www.elasticsearch.org/contributor-agreement/). We are not asking you to assign copyright to us, but to give us the right to distribute your code without restriction. We ask this of all contributors in order to assure our users of the origin and continuing existence of the code. You only need to sign the CLA once. + +3. Rebase your changes +Update your local repository with the most recent code from the main Elasticsearch repository, and rebase your branch on top of the latest master branch. We prefer your changes to be squashed into a single commit. + +4. Submit a pull request +Push your local changes to your forked copy of the repository and [submit a pull request](https://help.github.com/articles/using-pull-requests). In the pull request, describe what your changes do and mention the number of the issue where discussion has taken place, eg "Closes #123". + +Then sit back and wait. There will probably be discussion about the pull request and, if any changes are needed, we would love to work with you to get your pull request merged into Elasticsearch. + + +Contributing to the Elasticsearch plugin +---------------------------------------- + +**Repository:** [https://github.com/elasticsearch/elasticsearch-lang-python](https://github.com/elasticsearch/elasticsearch-lang-python) + +Make sure you have [Maven](http://maven.apache.org) installed, as Elasticsearch uses it as its build system. Integration with IntelliJ and Eclipse should work out of the box. Eclipse users can automatically configure their IDE by running `mvn eclipse:eclipse` and then importing the project into their workspace: `File > Import > Existing project into workspace`. + +Please follow these formatting guidelines: + +* Java indent is 4 spaces +* Line width is 140 characters +* The rest is left to Java coding standards +* Disable “auto-format on save” to prevent unnecessary format changes. This makes reviews much harder as it generates unnecessary formatting changes. If your IDE supports formatting only modified chunks that is fine to do. + +To create a distribution from the source, simply run: + +```sh +cd elasticsearch-lang-python/ +mvn clean package -DskipTests +``` + +You will find the newly built packages under: `./target/releases/`. + +Before submitting your changes, run the test suite to make sure that nothing is broken, with: + +```sh +mvn clean test +``` + +Source: [Contributing to elasticsearch](http://www.elasticsearch.org/contributing-to-elasticsearch/) From 9d431481e389b2dfa77b31493cfddaa0dfc397ac Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 14 Jan 2014 16:53:50 +0100 Subject: [PATCH 14/73] update headers --- README.md | 21 +++++++------------ .../plugin/python/PythonPlugin.java | 14 ++++++------- .../python/PythonScriptEngineService.java | 14 ++++++------- .../python/PythonScriptEngineTests.java | 14 ++++++------- .../python/PythonScriptMultiThreadedTest.java | 14 ++++++------- .../python/PythonScriptSearchTests.java | 14 ++++++------- .../script/python/SimpleBench.java | 14 ++++++------- 7 files changed, 50 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index d0769cc32fd..57ea805beed 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,23 @@ -Python lang Plugin for ElasticSearch +Python lang Plugin for Elasticsearch ================================== The Python (jython) language plugin allows to have `python` as the language of scripts to execute. In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/1.2.0`. - --------------------------------------- - | Python Plugin | ElasticSearch | - --------------------------------------- - | master | 0.90 -> master | - --------------------------------------- - | 1.2.0 | 0.90 -> master | - --------------------------------------- - | 1.1.0 | 0.19 -> 0.20 | - --------------------------------------- - | 1.0.0 | 0.18 | - --------------------------------------- +| Python Lang Plugin | elasticsearch | jython |Release date | +|----------------------------|---------------------|----------|:------------:| +| 1.3.0-SNAPSHOT | 0.90 | 2.5.2 | | +| 1.2.0 | 0.90 | 2.5.2 | 2013-02-26 | +| 1.1.0 | 0.19 -> 0.20 | 2.5.2 | 2012-02-07 | +| 1.0.0 | 0.18 | 2.5.2 | 2011-12-05 | License ------- This software is licensed under the Apache 2 license, quoted below. - Copyright 2009-2012 Shay Banon and ElasticSearch + Copyright 2009-2014 Elasticsearch Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of diff --git a/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java b/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java index 6a54b8a1236..1e82ea61851 100644 --- a/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java +++ b/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java index 877c6f50c2a..16da9eba698 100644 --- a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java index f2db686efe8..cce5728b25b 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java b/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java index 01488a43708..2caa518d4ea 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java index 91038aa94b3..4b7cd282748 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/src/test/java/org/elasticsearch/script/python/SimpleBench.java b/src/test/java/org/elasticsearch/script/python/SimpleBench.java index 362e838a9f0..7877187c866 100644 --- a/src/test/java/org/elasticsearch/script/python/SimpleBench.java +++ b/src/test/java/org/elasticsearch/script/python/SimpleBench.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * From de2ae17633520ece87ff3248429606d16eba38c8 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 14 Jan 2014 16:57:08 +0100 Subject: [PATCH 15/73] Update to jython 2.5.3 Closes #7. --- README.md | 2 +- pom.xml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 57ea805beed..3ecc5d62dc1 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e | Python Lang Plugin | elasticsearch | jython |Release date | |----------------------------|---------------------|----------|:------------:| -| 1.3.0-SNAPSHOT | 0.90 | 2.5.2 | | +| 1.3.0-SNAPSHOT | 0.90 | 2.5.3 | | | 1.2.0 | 0.90 | 2.5.2 | 2013-02-26 | | 1.1.0 | 0.19 -> 0.20 | 2.5.2 | 2012-02-07 | | 1.0.0 | 0.18 | 2.5.2 | 2011-12-05 | diff --git a/pom.xml b/pom.xml index f782217b230..2c7aa5461f9 100644 --- a/pom.xml +++ b/pom.xml @@ -51,10 +51,8 @@ org.python jython-standalone - 2.5.2 + 2.5.3 compile - - From c065e3884b572c4c73ed2773acc1e927878e86d9 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 14 Jan 2014 17:48:09 +0100 Subject: [PATCH 16/73] Move tests to elasticsearch test framework Closes #6. --- .gitignore | 1 + pom.xml | 112 ++++++++++++- .../python/PythonScriptEngineTests.java | 19 ++- .../python/PythonScriptMultiThreadedTest.java | 17 +- .../python/PythonScriptSearchTests.java | 147 ++++++------------ 5 files changed, 177 insertions(+), 119 deletions(-) diff --git a/.gitignore b/.gitignore index 06a1e6fedb6..21d2106f97c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ /target .DS_Store *.iml +.local-execution-hints.log diff --git a/pom.xml b/pom.xml index 2c7aa5461f9..b6d5fd71a3b 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,13 @@ - 0.90.0.Beta1 + 0.90.10 + 4.6.0 + 1 + true + onerror + + INFO @@ -41,6 +47,13 @@ + + org.apache.lucene + lucene-test-framework + ${lucene.version} + test + + org.elasticsearch elasticsearch @@ -63,9 +76,10 @@ - org.testng - testng - 6.3.1 + org.elasticsearch + elasticsearch + ${elasticsearch.version} + test-jar test @@ -96,13 +110,95 @@ + com.carrotsearch.randomizedtesting + junit4-maven-plugin + 2.0.12 + + + tests + test + + junit4 + + + 20 + pipe,warn + true + + + + + + + + + ${tests.jvms} + + + + + + + **/*Tests.class + **/*Test.class + + + **/Abstract*.class + **/*StressTest.class + + + -Xmx512m + -XX:MaxDirectMemorySize=512m + -Des.logger.prefix= + + ${tests.shuffle} + ${tests.verbose} + ${tests.seed} + ${tests.failfast} + + + ${tests.iters} + ${tests.maxfailures} + ${tests.failfast} + ${tests.class} + ${tests.method} + ${tests.nightly} + ${tests.badapples} + ${tests.weekly} + ${tests.slow} + ${tests.awaitsfix} + ${tests.slow} + ${tests.timeoutSuite} + ${tests.showSuccess} + ${tests.integration} + ${tests.cluster_seed} + ${tests.client.ratio} + ${env.ES_TEST_LOCAL} + ${es.node.mode} + ${es.logger.level} + true + + + + + + + org.apache.maven.plugins maven-surefire-plugin - 2.11 + 2.15 - - **/*Tests.java - + true diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java index cce5728b25b..92435ea9be3 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java @@ -23,32 +23,34 @@ import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.script.ExecutableScript; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; +import org.elasticsearch.test.ElasticsearchTestCase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; import java.util.HashMap; import java.util.Map; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; /** * */ -@Test -public class PythonScriptEngineTests { +public class PythonScriptEngineTests extends ElasticsearchTestCase { private PythonScriptEngineService se; - @BeforeClass + @Before public void setup() { se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS); } - @AfterClass + @After public void close() { + // We need to clear some system properties + System.clearProperty("python.cachedir.skip"); + System.clearProperty("python.console.encoding"); se.close(); } @@ -95,6 +97,7 @@ public class PythonScriptEngineTests { @Test public void testAccessListInScript() { + Map vars = new HashMap(); Map obj2 = MapBuilder.newMapBuilder().put("prop2", "value2").map(); Map obj1 = MapBuilder.newMapBuilder().put("prop1", "value1").put("obj2", obj2).map(); diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java b/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java index 2caa518d4ea..0af0505eeca 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java @@ -19,12 +19,12 @@ package org.elasticsearch.script.python; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.util.concurrent.jsr166y.ThreadLocalRandom; import org.elasticsearch.script.ExecutableScript; -import org.testng.annotations.Test; +import org.elasticsearch.test.ElasticsearchTestCase; +import org.junit.After; +import org.junit.Test; import java.util.HashMap; import java.util.Map; @@ -32,16 +32,19 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; /** * */ -@Test -public class PythonScriptMultiThreadedTest { +public class PythonScriptMultiThreadedTest extends ElasticsearchTestCase { - protected final ESLogger logger = Loggers.getLogger(getClass()); + @After + public void close() { + // We need to clear some system properties + System.clearProperty("python.cachedir.skip"); + System.clearProperty("python.console.encoding"); + } @Test public void testExecutableNoRuntimeParams() throws Exception { diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java index 4b7cd282748..642658d9b1c 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java @@ -21,17 +21,10 @@ package org.elasticsearch.script.python; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.Loggers; -import org.elasticsearch.common.network.NetworkUtils; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; import org.elasticsearch.search.sort.SortOrder; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; +import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.junit.After; +import org.junit.Test; import java.util.Arrays; import java.util.List; @@ -42,55 +35,33 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.FilterBuilders.scriptFilter; import static org.elasticsearch.index.query.QueryBuilders.*; import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; /** * */ -@Test -public class PythonScriptSearchTests { +public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { - protected final ESLogger logger = Loggers.getLogger(getClass()); - - private Node node; - - private Client client; - - @BeforeMethod - public void createNodes() throws Exception { - node = NodeBuilder.nodeBuilder().settings(ImmutableSettings.settingsBuilder() - .put("path.data", "target/data") - .put("cluster.name", "test-cluster-" + NetworkUtils.getLocalAddress()) - .put("gateway.type", "none") - .put("number_of_shards", 1)).node(); - client = node.client(); - } - - @AfterMethod - public void closeNodes() { - client.close(); - node.close(); + @After + public void close() { + // We need to clear some system properties + System.clearProperty("python.cachedir.skip"); + System.clearProperty("python.console.encoding"); } @Test public void testPythonFilter() throws Exception { - client.admin().indices().prepareCreate("test").execute().actionGet(); - client.prepareIndex("test", "type1", "1") - .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject()) - .execute().actionGet(); - client.admin().indices().prepareFlush().execute().actionGet(); - client.prepareIndex("test", "type1", "2") - .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject()) - .execute().actionGet(); - client.admin().indices().prepareFlush().execute().actionGet(); - client.prepareIndex("test", "type1", "3") - .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).endObject()) - .execute().actionGet(); - client.admin().indices().refresh(refreshRequest()).actionGet(); + wipeIndices("test"); + createIndex("test"); + index("test", "type1", "1", jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject()); + flush(); + index("test", "type1", "2", jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject()); + flush(); + index("test", "type1", "3", jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).endObject()); + refresh(); - logger.info("running doc['num1'].value > 1"); - SearchResponse response = client.prepareSearch() + logger.info(" --> running doc['num1'].value > 1"); + SearchResponse response = client().prepareSearch() .setQuery(filteredQuery(matchAllQuery(), scriptFilter("doc['num1'].value > 1").lang("python"))) .addSort("num1", SortOrder.ASC) .addScriptField("sNum1", "python", "doc['num1'].value", null) @@ -102,8 +73,8 @@ public class PythonScriptSearchTests { assertThat(response.getHits().getAt(1).id(), equalTo("3")); assertThat((Double) response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(3.0)); - logger.info("running doc['num1'].value > param1"); - response = client.prepareSearch() + logger.info(" --> running doc['num1'].value > param1"); + response = client().prepareSearch() .setQuery(filteredQuery(matchAllQuery(), scriptFilter("doc['num1'].value > param1").lang("python").addParam("param1", 2))) .addSort("num1", SortOrder.ASC) .addScriptField("sNum1", "python", "doc['num1'].value", null) @@ -113,8 +84,8 @@ public class PythonScriptSearchTests { assertThat(response.getHits().getAt(0).id(), equalTo("3")); assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(3.0)); - logger.info("running doc['num1'].value > param1"); - response = client.prepareSearch() + logger.info(" --> running doc['num1'].value > param1"); + response = client().prepareSearch() .setQuery(filteredQuery(matchAllQuery(), scriptFilter("doc['num1'].value > param1").lang("python").addParam("param1", -1))) .addSort("num1", SortOrder.ASC) .addScriptField("sNum1", "python", "doc['num1'].value", null) @@ -131,16 +102,16 @@ public class PythonScriptSearchTests { @Test public void testScriptFieldUsingSource() throws Exception { - client.admin().indices().prepareCreate("test").execute().actionGet(); - client.prepareIndex("test", "type1", "1") - .setSource(jsonBuilder().startObject() + wipeIndices("test"); + createIndex("test"); + index("test", "type1", "1", + jsonBuilder().startObject() .startObject("obj1").field("test", "something").endObject() .startObject("obj2").startArray("arr2").value("arr_value1").value("arr_value2").endArray().endObject() - .endObject()) - .execute().actionGet(); - client.admin().indices().refresh(refreshRequest()).actionGet(); + .endObject()); + refresh(); - SearchResponse response = client.prepareSearch() + SearchResponse response = client().prepareSearch() .setQuery(matchAllQuery()) .addField("_source.obj1") // we also automatically detect _source in fields .addScriptField("s_obj1", "python", "_source['obj1']", null) @@ -171,32 +142,16 @@ public class PythonScriptSearchTests { @Test public void testCustomScriptBoost() throws Exception { - // execute a search before we create an index - try { - client.prepareSearch().setQuery(termQuery("test", "value")).execute().actionGet(); - assert false : "should fail"; - } catch (Exception e) { - // ignore, no indices - } - - try { - client.prepareSearch("test").setQuery(termQuery("test", "value")).execute().actionGet(); - assert false : "should fail"; - } catch (Exception e) { - // ignore, no indices - } - - client.admin().indices().create(createIndexRequest("test")).actionGet(); - client.index(indexRequest("test").type("type1").id("1") - .source(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject())).actionGet(); - client.index(indexRequest("test").type("type1").id("2") - .source(jsonBuilder().startObject().field("test", "value check").field("num1", 2.0f).endObject())).actionGet(); - client.admin().indices().refresh(refreshRequest()).actionGet(); + wipeIndices("test"); + createIndex("test"); + index("test", "type1", "1", jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject()); + index("test", "type1", "2", jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject()); + refresh(); logger.info("--- QUERY_THEN_FETCH"); - logger.info("running doc['num1'].value"); - SearchResponse response = client.search(searchRequest() + logger.info(" --> running doc['num1'].value"); + SearchResponse response = client().search(searchRequest() .searchType(SearchType.QUERY_THEN_FETCH) .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("doc['num1'].value").lang("python"))) ).actionGet(); @@ -204,13 +159,13 @@ public class PythonScriptSearchTests { assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); assertThat(response.getHits().totalHits(), equalTo(2l)); - logger.info("Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); - logger.info("Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); assertThat(response.getHits().getAt(0).id(), equalTo("2")); assertThat(response.getHits().getAt(1).id(), equalTo("1")); - logger.info("running -doc['num1'].value"); - response = client.search(searchRequest() + logger.info(" --> running -doc['num1'].value"); + response = client().search(searchRequest() .searchType(SearchType.QUERY_THEN_FETCH) .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("-doc['num1'].value").lang("python"))) ).actionGet(); @@ -218,14 +173,14 @@ public class PythonScriptSearchTests { assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); assertThat(response.getHits().totalHits(), equalTo(2l)); - logger.info("Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); - logger.info("Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); assertThat(response.getHits().getAt(0).id(), equalTo("1")); assertThat(response.getHits().getAt(1).id(), equalTo("2")); - logger.info("running doc['num1'].value * _score"); - response = client.search(searchRequest() + logger.info(" --> running doc['num1'].value * _score"); + response = client().search(searchRequest() .searchType(SearchType.QUERY_THEN_FETCH) .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("doc['num1'].value * _score").lang("python"))) ).actionGet(); @@ -233,13 +188,13 @@ public class PythonScriptSearchTests { assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); assertThat(response.getHits().totalHits(), equalTo(2l)); - logger.info("Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); - logger.info("Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); assertThat(response.getHits().getAt(0).id(), equalTo("2")); assertThat(response.getHits().getAt(1).id(), equalTo("1")); - logger.info("running param1 * param2 * _score"); - response = client.search(searchRequest() + logger.info(" --> running param1 * param2 * _score"); + response = client().search(searchRequest() .searchType(SearchType.QUERY_THEN_FETCH) .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("param1 * param2 * _score").param("param1", 2).param("param2", 2).lang("python"))) ).actionGet(); @@ -247,7 +202,7 @@ public class PythonScriptSearchTests { assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); assertThat(response.getHits().totalHits(), equalTo(2l)); - logger.info("Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); - logger.info("Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); } } From 3fc121f603eae5c3c12f9107a38479a152999d8d Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 14 Jan 2014 19:09:10 +0100 Subject: [PATCH 17/73] Update request that uses python script with no parameters fails with NullPointerException Closes #4. --- .../python/PythonScriptEngineService.java | 6 +++-- .../python/PythonScriptSearchTests.java | 26 ++++++++++++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java index 16da9eba698..2a94949830d 100644 --- a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java @@ -107,8 +107,10 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri public PythonExecutableScript(PyCode code, Map vars) { this.code = code; this.pyVars = new PyStringMap(); - for (Map.Entry entry : vars.entrySet()) { - pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue())); + if (vars != null) { + for (Map.Entry entry : vars.entrySet()) { + pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue())); + } } } diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java index 642658d9b1c..2f90291d651 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.hamcrest.CoreMatchers; import org.junit.After; import org.junit.Test; @@ -30,11 +31,12 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import static org.elasticsearch.client.Requests.*; +import static org.elasticsearch.client.Requests.searchRequest; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.FilterBuilders.scriptFilter; import static org.elasticsearch.index.query.QueryBuilders.*; import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; +import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.equalTo; /** @@ -205,4 +207,26 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); } + + /** + * Test case for #4: https://github.com/elasticsearch/elasticsearch-lang-python/issues/4 + * Update request that uses python script with no parameters fails with NullPointerException + * @throws Exception + */ + @Test + public void testPythonEmptyParameters() throws Exception { + wipeIndices("test"); + createIndex("test"); + index("test", "type1", "1", jsonBuilder().startObject().field("myfield", "foo").endObject()); + refresh(); + + client().prepareUpdate("test", "type1", "1").setScriptLang("python").setScript("ctx[\"_source\"][\"myfield\"]=\"bar\"") + .execute().actionGet(); + refresh(); + + Object value = get("test", "type1", "1").getSourceAsMap().get("myfield"); + assertThat(value instanceof String, is(true)); + + assertThat((String) value, CoreMatchers.equalTo("bar")); + } } From 20192eb678821a8d1e320600798cc3d36722225d Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 14 Jan 2014 19:11:23 +0100 Subject: [PATCH 18/73] prepare branch 1.x --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3ecc5d62dc1..3684a886ad1 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,13 @@ The Python (jython) language plugin allows to have `python` as the language of s In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/1.2.0`. -| Python Lang Plugin | elasticsearch | jython |Release date | -|----------------------------|---------------------|----------|:------------:| -| 1.3.0-SNAPSHOT | 0.90 | 2.5.3 | | -| 1.2.0 | 0.90 | 2.5.2 | 2013-02-26 | -| 1.1.0 | 0.19 -> 0.20 | 2.5.2 | 2012-02-07 | -| 1.0.0 | 0.18 | 2.5.2 | 2011-12-05 | +| Python Lang Plugin | elasticsearch | jython |Release date | +|-----------------------------|---------------------|----------|:------------:| +| 2.0.0.RC1-SNAPSHOT (master) | 1.0.0.RC1 -> master | 2.5.3 | | +| 1.3.0-SNAPSHOT (1.x) | 0.90 | 2.5.3 | | +| 1.2.0 | 0.90 | 2.5.2 | 2013-02-26 | +| 1.1.0 | 0.19 -> 0.20 | 2.5.2 | 2012-02-07 | +| 1.0.0 | 0.18 | 2.5.2 | 2011-12-05 | License ------- From f5865b71af7b18c2089540a78abeff738d8a6242 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 14 Jan 2014 19:16:19 +0100 Subject: [PATCH 19/73] Update to elasticsearch 1.0.0.RC1 Closes #5. --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index b6d5fd71a3b..40f8b03cd74 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 4.0.0 org.elasticsearch elasticsearch-lang-python - 1.3.0-SNAPSHOT + 2.0.0.RC1-SNAPSHOT jar JavaScript lang plugin for ElasticSearch 2009 @@ -30,7 +30,7 @@ - 0.90.10 + 1.0.0.RC1-SNAPSHOT 4.6.0 1 true From 5a43c62d3bb0472974d72af8f03bb8dee4964284 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 15 Jan 2014 18:47:41 +0100 Subject: [PATCH 20/73] prepare release elasticsearch-lang-python-2.0.0.RC1 --- README.md | 7 ++++--- pom.xml | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3684a886ad1..a4e0e5272fa 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,12 @@ Python lang Plugin for Elasticsearch The Python (jython) language plugin allows to have `python` as the language of scripts to execute. -In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/1.2.0`. +In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/2.0.0.RC1`. -| Python Lang Plugin | elasticsearch | jython |Release date | +| Python Lang Plugin | elasticsearch | jython | Release date | |-----------------------------|---------------------|----------|:------------:| -| 2.0.0.RC1-SNAPSHOT (master) | 1.0.0.RC1 -> master | 2.5.3 | | +| 2.0.0-SNAPSHOT (master) | 1.0.0.RC1 -> master | 2.5.3 | | +| 2.0.0.RC1 | 1.0.0.RC1 -> master | 2.5.3 | 2014-01-15 | | 1.3.0-SNAPSHOT (1.x) | 0.90 | 2.5.3 | | | 1.2.0 | 0.90 | 2.5.2 | 2013-02-26 | | 1.1.0 | 0.19 -> 0.20 | 2.5.2 | 2012-02-07 | diff --git a/pom.xml b/pom.xml index 40f8b03cd74..2c2ba1adfeb 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 4.0.0 org.elasticsearch elasticsearch-lang-python - 2.0.0.RC1-SNAPSHOT + 2.0.0.RC1 jar JavaScript lang plugin for ElasticSearch 2009 @@ -30,7 +30,7 @@ - 1.0.0.RC1-SNAPSHOT + 1.0.0.RC1 4.6.0 1 true From b7511a18caefecb91c6ae8bf974f75e33bda956d Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 15 Jan 2014 19:54:57 +0100 Subject: [PATCH 21/73] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2c2ba1adfeb..e4e0e8d17b0 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 4.0.0 org.elasticsearch elasticsearch-lang-python - 2.0.0.RC1 + 2.0.0-SNAPSHOT jar JavaScript lang plugin for ElasticSearch 2009 From 79ac03e9c55fd5c31e44bdb67a2193b18f1e3416 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 7 Mar 2014 17:17:47 +0100 Subject: [PATCH 22/73] Add plugin release semi-automatic script Closes #8. --- README.md | 9 +- dev-tools/build_release.py | 708 +++++++++++++++++++++++++++++++++++++ dev-tools/upload-s3.py | 67 ++++ pom.xml | 5 +- 4 files changed, 782 insertions(+), 7 deletions(-) create mode 100755 dev-tools/build_release.py create mode 100644 dev-tools/upload-s3.py diff --git a/README.md b/README.md index a4e0e5272fa..f07a6fe296f 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,13 @@ The Python (jython) language plugin allows to have `python` as the language of s In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/2.0.0.RC1`. +* For 1.0.x elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/master). +* For 0.90.x elasticsearch versions, look at [1.x branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/1.x). + | Python Lang Plugin | elasticsearch | jython | Release date | |-----------------------------|---------------------|----------|:------------:| -| 2.0.0-SNAPSHOT (master) | 1.0.0.RC1 -> master | 2.5.3 | | +| 2.0.0-SNAPSHOT | 1.0.0.RC1 -> master | 2.5.3 | XXXX-XX-XX | | 2.0.0.RC1 | 1.0.0.RC1 -> master | 2.5.3 | 2014-01-15 | -| 1.3.0-SNAPSHOT (1.x) | 0.90 | 2.5.3 | | -| 1.2.0 | 0.90 | 2.5.2 | 2013-02-26 | -| 1.1.0 | 0.19 -> 0.20 | 2.5.2 | 2012-02-07 | -| 1.0.0 | 0.18 | 2.5.2 | 2011-12-05 | License ------- diff --git a/dev-tools/build_release.py b/dev-tools/build_release.py new file mode 100755 index 00000000000..9166b09e7e3 --- /dev/null +++ b/dev-tools/build_release.py @@ -0,0 +1,708 @@ +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on +# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import re +import tempfile +import shutil +import os +import datetime +import argparse +import github3 +import smtplib + +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +from os.path import dirname, abspath + +""" + This tool builds a release from the a given elasticsearch plugin branch. + In order to execute it go in the top level directory and run: + $ python3 dev_tools/build_release.py --branch master --publish --remote origin + + By default this script runs in 'dry' mode which essentially simulates a release. If the + '--publish' option is set the actual release is done. + If not in 'dry' mode, a mail will be automatically sent to the mailing list. + You can disable it with the option '--disable_mail' + + $ python3 dev_tools/build_release.py --publish --remote origin --disable_mail + + The script takes over almost all + steps necessary for a release from a high level point of view it does the following things: + + - run prerequisite checks ie. check for Java 1.6 being present or S3 credentials available as env variables + - detect the version to release from the specified branch (--branch) or the current branch + - creates a release branch & updates pom.xml and README.md to point to a release version rather than a snapshot + - builds the artifacts + - commits the new version and merges the release branch into the source branch + - creates a tag and pushes the commit to the specified origin (--remote) + - publishes the releases to sonatype and S3 + - send a mail based on github issues fixed by this version + +Once it's done it will print all the remaining steps. + + Prerequisites: + - Python 3k for script execution + - Boto for S3 Upload ($ apt-get install python-boto or pip-3.3 install boto) + - github3 module (pip-3.3 install github3.py) + - S3 keys exported via ENV Variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) + - GITHUB (login/password) or key exported via ENV Variables (GITHUB_LOGIN, GITHUB_PASSWORD or GITHUB_KEY) + (see https://github.com/settings/applications#personal-access-tokens) - Optional: default to no authentication + - SMTP_HOST - Optional: default to localhost + - MAIL_SENDER - Optional: default to 'david@pilato.fr': must be authorized to send emails to elasticsearch mailing list + - MAIL_TO - Optional: default to 'elasticsearch@googlegroups.com' +""" +env = os.environ + +LOG = env.get('ES_RELEASE_LOG', '/tmp/elasticsearch_release.log') +ROOT_DIR = os.path.join(abspath(dirname(__file__)), '../') +README_FILE = ROOT_DIR + 'README.md' +POM_FILE = ROOT_DIR + 'pom.xml' + +def log(msg): + log_plain('\n%s' % msg) + +def log_plain(msg): + f = open(LOG, mode='ab') + f.write(msg.encode('utf-8')) + f.close() + +def run(command, quiet=False): + log('%s: RUN: %s\n' % (datetime.datetime.now(), command)) + if os.system('%s >> %s 2>&1' % (command, LOG)): + msg = ' FAILED: %s [see log %s]' % (command, LOG) + if not quiet: + print(msg) + raise RuntimeError(msg) + +try: + JAVA_HOME = env['JAVA_HOME'] +except KeyError: + raise RuntimeError(""" + Please set JAVA_HOME in the env before running release tool + On OSX use: export JAVA_HOME=`/usr/libexec/java_home -v '1.6*'`""") + +try: + MVN='mvn' + # make sure mvn3 is used if mvn3 is available + # some systems use maven 2 as default + run('mvn3 --version', quiet=True) + MVN='mvn3' +except RuntimeError: + pass + + +def java_exe(): + path = JAVA_HOME + return 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % (path, path, path) + +def verify_java_version(version): + s = os.popen('%s; java -version 2>&1' % java_exe()).read() + if s.find(' version "%s.' % version) == -1: + raise RuntimeError('got wrong version for java %s:\n%s' % (version, s)) + +# Verifies the java version. We guarantee that we run with Java 1.6 +# If 1.6 is not available fail the build! +def verify_mvn_java_version(version, mvn): + s = os.popen('%s; %s --version 2>&1' % (java_exe(), mvn)).read() + if s.find('Java version: %s' % version) == -1: + raise RuntimeError('got wrong java version for %s %s:\n%s' % (mvn, version, s)) + +# Returns the hash of the current git HEAD revision +def get_head_hash(): + return os.popen(' git rev-parse --verify HEAD 2>&1').read().strip() + +# Returns the hash of the given tag revision +def get_tag_hash(tag): + return os.popen('git show-ref --tags %s --hash 2>&1' % (tag)).read().strip() + +# Returns the name of the current branch +def get_current_branch(): + return os.popen('git rev-parse --abbrev-ref HEAD 2>&1').read().strip() + +verify_java_version('1.6') # we require to build with 1.6 +verify_mvn_java_version('1.6', MVN) + +# Utility that returns the name of the release branch for a given version +def release_branch(version): + return 'release_branch_%s' % version + +# runs get fetch on the given remote +def fetch(remote): + run('git fetch %s' % remote) + +# Creates a new release branch from the given source branch +# and rebases the source branch from the remote before creating +# the release branch. Note: This fails if the source branch +# doesn't exist on the provided remote. +def create_release_branch(remote, src_branch, release): + run('git checkout %s' % src_branch) + run('git pull --rebase %s %s' % (remote, src_branch)) + run('git checkout -b %s' % (release_branch(release))) + + +# Reads the given file and applies the +# callback to it. If the callback changed +# a line the given file is replaced with +# the modified input. +def process_file(file_path, line_callback): + fh, abs_path = tempfile.mkstemp() + modified = False + with open(abs_path,'w', encoding='utf-8') as new_file: + with open(file_path, encoding='utf-8') as old_file: + for line in old_file: + new_line = line_callback(line) + modified = modified or (new_line != line) + new_file.write(new_line) + os.close(fh) + if modified: + #Remove original file + os.remove(file_path) + #Move new file + shutil.move(abs_path, file_path) + return True + else: + # nothing to do - just remove the tmp file + os.remove(abs_path) + return False + +# Guess the next snapshot version number (increment second digit) +def guess_snapshot(version): + digits=list(map(int, re.findall(r'\d+', version))) + source='%s.%s' % (digits[0], digits[1]) + destination='%s.%s' % (digits[0], digits[1]+1) + return version.replace(source, destination) + +# Moves the pom.xml file from a snapshot to a release +def remove_maven_snapshot(pom, release): + pattern = '%s-SNAPSHOT' % release + replacement = '%s' % release + def callback(line): + return line.replace(pattern, replacement) + process_file(pom, callback) + +# Moves the README.md file from a snapshot to a release +def remove_version_snapshot(readme_file, release): + pattern = '%s-SNAPSHOT' % release + replacement = '%s ' % release + def callback(line): + return line.replace(pattern, replacement) + process_file(readme_file, callback) + +# Moves the pom.xml file to the next snapshot +def add_maven_snapshot(pom, release, snapshot): + pattern = '%s' % release + replacement = '%s-SNAPSHOT' % snapshot + def callback(line): + return line.replace(pattern, replacement) + process_file(pom, callback) + +# Add in README.md file the next snapshot +def add_version_snapshot(readme_file, release, snapshot): + pattern = '| %s ' % release + replacement = '| %s-SNAPSHOT' % snapshot + def callback(line): + # If we find pattern, we copy the line and replace its content + if line.find(pattern) >= 0: + return line.replace(pattern, replacement).replace('%s' % (datetime.datetime.now().strftime("%Y-%m-%d")), + 'XXXX-XX-XX')+line + else: + return line + process_file(readme_file, callback) + + +# Set release date in README.md file +def set_date(readme_file): + pattern = 'XXXX-XX-XX' + replacement = '%s' % (datetime.datetime.now().strftime("%Y-%m-%d")) + def callback(line): + return line.replace(pattern, replacement) + process_file(readme_file, callback) + +# Update installation instructions in README.md file +def set_install_instructions(readme_file, artifact_name, release): + pattern = '`bin/plugin -install elasticsearch/%s/.+`' % artifact_name + replacement = '`bin/plugin -install elasticsearch/%s/%s`' % (artifact_name, release) + def callback(line): + return re.sub(pattern, replacement, line) + process_file(readme_file, callback) + + +# Stages the given files for the next git commit +def add_pending_files(*files): + for file in files: + run('git add %s' % file) + +# Executes a git commit with 'release [version]' as the commit message +def commit_release(artifact_id, release): + run('git commit -m "prepare release %s-%s"' % (artifact_id, release)) + +def commit_snapshot(): + run('git commit -m "prepare for next development iteration"') + +def tag_release(release): + run('git tag -a v%s -m "Tag release version %s"' % (release, release)) + +def run_mvn(*cmd): + for c in cmd: + run('%s; %s -f %s %s' % (java_exe(), MVN, POM_FILE, c)) + +def build_release(run_tests=False, dry_run=True): + target = 'deploy' + if dry_run: + target = 'package' + if run_tests: + run_mvn('clean test') + run_mvn('clean %s -DskipTests' %(target)) + +# Checks the pom.xml for the release version. 2.0.0-SNAPSHOT +# This method fails if the pom file has no SNAPSHOT version set ie. +# if the version is already on a release version we fail. +# Returns the next version string ie. 0.90.7 +def find_release_version(src_branch): + run('git checkout %s' % src_branch) + with open(POM_FILE, encoding='utf-8') as file: + for line in file: + match = re.search(r'(.+)-SNAPSHOT', line) + if match: + return match.group(1) + raise RuntimeError('Could not find release version in branch %s' % src_branch) + +# extract a value from pom.xml +def find_from_pom(tag): + with open(POM_FILE, encoding='utf-8') as file: + for line in file: + match = re.search(r'<%s>(.+)' % (tag, tag), line) + if match: + return match.group(1) + raise RuntimeError('Could not find <%s> in pom.xml file' % (tag)) + +def get_artifacts(artifact_id, release): + artifact_path = ROOT_DIR + 'target/releases/%s-%s.zip' % (artifact_id, release) + print(' Path %s' % (artifact_path)) + if not os.path.isfile(artifact_path): + raise RuntimeError('Could not find required artifact at %s' % (artifact_path)) + return artifact_path + +# Generates sha1 for a file +# and returns the checksum files as well +# as the given files in a list +def generate_checksums(release_file): + res = [] + directory = os.path.dirname(release_file) + file = os.path.basename(release_file) + checksum_file = '%s.sha1.txt' % file + + if os.system('cd %s; shasum %s > %s' % (directory, file, checksum_file)): + raise RuntimeError('Failed to generate checksum for file %s' % release_file) + res = res + [os.path.join(directory, checksum_file), release_file] + return res + +def git_merge(src_branch, release_version): + run('git checkout %s' % src_branch) + run('git merge %s' % release_branch(release_version)) + +def git_push(remote, src_branch, release_version, dry_run): + if not dry_run: + run('git push %s %s' % (remote, src_branch)) # push the commit + run('git push %s v%s' % (remote, release_version)) # push the tag + else: + print(' dryrun [True] -- skipping push to remote %s' % remote) + +def publish_artifacts(artifacts, base='elasticsearch/elasticsearch', dry_run=True): + location = os.path.dirname(os.path.realpath(__file__)) + for artifact in artifacts: + if dry_run: + print('Skip Uploading %s to Amazon S3 in %s' % (artifact, base)) + else: + print('Uploading %s to Amazon S3' % artifact) + # requires boto to be installed but it is not available on python3k yet so we use a dedicated tool + run('python %s/upload-s3.py --file %s --path %s' % (location, os.path.abspath(artifact), base)) + + +################# +## +## +## Email and Github Management +## +## +################# +def format_issues_plain(issues, title='Fix'): + response = "" + + if len(issues) > 0: + response += '%s:\n' % title + for issue in issues: + response += ' * [%s] - %s (%s)\n' % (issue.number, issue.title, issue.html_url) + + return response + +def format_issues_html(issues, title='Fix'): + response = "" + + if len(issues) > 0: + response += '

%s

\n
    \n' % title + for issue in issues: + response += '[%s] - %s\n' % (issue.html_url, issue.number, issue.title) + response += '
\n' + + return response + +def get_github_repository(reponame, + login=env.get('GITHUB_LOGIN', None), + password=env.get('GITHUB_PASSWORD', None), + key=env.get('GITHUB_KEY', None)): + if login: + g = github3.login(login, password) + elif key: + g = github3.login(token=key) + else: + g = github3.GitHub() + + return g.repository("elasticsearch", reponame) + +# Check if there are some remaining open issues and fails +def check_opened_issues(version, repository, reponame): + opened_issues = [i for i in repository.iter_issues(state='open', labels='%s' % version)] + if len(opened_issues)>0: + raise NameError('Some issues [%s] are still opened. Check https://github.com/elasticsearch/%s/issues?labels=%s&state=open' + % (len(opened_issues), reponame, version)) + +# List issues from github: can be done anonymously if you don't +# exceed a given number of github API calls per day +# Check if there are some remaining open issues and fails +def list_issues(version, + repository, + severity='bug'): + issues = [i for i in repository.iter_issues(state='closed', labels='%s,%s' % (severity, version))] + return issues + +# Get issues from github and generates a Plain/HTML Multipart email +# And send it if dry_run=False +def prepare_email(artifact_id, release_version, repository, + artifact_name, artifact_description, project_url, + severity_labels_bug='bug', + severity_labels_update='update', + severity_labels_new='new', + severity_labels_doc='doc'): + + ## Get bugs from github + issues_bug = list_issues(release_version, repository, severity=severity_labels_bug) + issues_update = list_issues(release_version, repository, severity=severity_labels_update) + issues_new = list_issues(release_version, repository, severity=severity_labels_new) + issues_doc = list_issues(release_version, repository, severity=severity_labels_doc) + + ## Format content to plain text + plain_issues_bug = format_issues_plain(issues_bug, 'Fix') + plain_issues_update = format_issues_plain(issues_update, 'Update') + plain_issues_new = format_issues_plain(issues_new, 'New') + plain_issues_doc = format_issues_plain(issues_doc, 'Doc') + + ## Format content to html + html_issues_bug = format_issues_html(issues_bug, 'Fix') + html_issues_update = format_issues_html(issues_update, 'Update') + html_issues_new = format_issues_html(issues_new, 'New') + html_issues_doc = format_issues_html(issues_doc, 'Doc') + + if len(issues_bug)+len(issues_update)+len(issues_new)+len(issues_doc) > 0: + plain_empty_message = "" + html_empty_message = "" + + else: + plain_empty_message = "No issue listed for this release" + html_empty_message = "

No issue listed for this release

" + + msg = MIMEMultipart('alternative') + msg['Subject'] = '[ANN] %s %s released' % (artifact_name, release_version) + text = """ +Heya, + + +We are pleased to announce the release of the %(artifact_name)s, version %(release_version)s. + +%(artifact_description)s. + +%(project_url)s + +Release Notes - %(artifact_id)s - Version %(release_version)s + +%(empty_message)s +%(issues_bug)s +%(issues_update)s +%(issues_new)s +%(issues_doc)s + +Issues, Pull requests, Feature requests are warmly welcome on %(artifact_id)s project repository: %(project_url)s +For questions or comments around this plugin, feel free to use elasticsearch mailing list: https://groups.google.com/forum/#!forum/elasticsearch + +Enjoy, + +-The Elasticsearch team +""" % {'release_version': release_version, + 'artifact_id': artifact_id, + 'artifact_name': artifact_name, + 'artifact_description': artifact_description, + 'project_url': project_url, + 'empty_message': plain_empty_message, + 'issues_bug': plain_issues_bug, + 'issues_update': plain_issues_update, + 'issues_new': plain_issues_new, + 'issues_doc': plain_issues_doc} + + html = """ + + +

Heya,

+ +

We are pleased to announce the release of the %(artifact_name)s, version %(release_version)s

+ +
%(artifact_description)s.
+ +

Release Notes - Version %(release_version)s

+%(empty_message)s +%(issues_bug)s +%(issues_update)s +%(issues_new)s +%(issues_doc)s + +

Issues, Pull requests, Feature requests are warmly welcome on +%(artifact_id)s project repository!

+

For questions or comments around this plugin, feel free to use elasticsearch +mailing list!

+ +

Enjoy,

+ +

- The Elasticsearch team

+ +""" % {'release_version': release_version, + 'artifact_id': artifact_id, + 'artifact_name': artifact_name, + 'artifact_description': artifact_description, + 'project_url': project_url, + 'empty_message': html_empty_message, + 'issues_bug': html_issues_bug, + 'issues_update': html_issues_update, + 'issues_new': html_issues_new, + 'issues_doc': html_issues_doc} + + # Record the MIME types of both parts - text/plain and text/html. + part1 = MIMEText(text, 'plain') + part2 = MIMEText(html, 'html') + + # Attach parts into message container. + # According to RFC 2046, the last part of a multipart message, in this case + # the HTML message, is best and preferred. + msg.attach(part1) + msg.attach(part2) + + return msg + +def send_email(msg, + dry_run=True, + mail=True, + sender=env.get('MAIL_SENDER'), + to=env.get('MAIL_TO', 'elasticsearch@googlegroups.com'), + smtp_server=env.get('SMTP_SERVER', 'localhost')): + msg['From'] = 'Elasticsearch Team <%s>' % sender + msg['To'] = 'Elasticsearch Mailing List <%s>' % to + # save mail on disk + with open(ROOT_DIR+'target/email.txt', 'w') as email_file: + email_file.write(msg.as_string()) + if mail and not dry_run: + s = smtplib.SMTP(smtp_server, 25) + s.sendmail(sender, to, msg.as_string()) + s.quit() + else: + print('generated email: open %starget/email.txt' % ROOT_DIR) + +def print_sonatype_notice(): + settings = os.path.join(os.path.expanduser('~'), '.m2/settings.xml') + if os.path.isfile(settings): + with open(settings, encoding='utf-8') as settings_file: + for line in settings_file: + if line.strip() == 'sonatype-nexus-snapshots': + # moving out - we found the indicator no need to print the warning + return + print(""" + NOTE: No sonatype settings detected, make sure you have configured + your sonatype credentials in '~/.m2/settings.xml': + + + ... + + + sonatype-nexus-snapshots + your-jira-id + your-jira-pwd + + + sonatype-nexus-staging + your-jira-id + your-jira-pwd + + + ... + + """) + +def check_s3_credentials(): + if not env.get('AWS_ACCESS_KEY_ID', None) or not env.get('AWS_SECRET_ACCESS_KEY', None): + raise RuntimeError('Could not find "AWS_ACCESS_KEY_ID" / "AWS_SECRET_ACCESS_KEY" in the env variables please export in order to upload to S3') + +def check_github_credentials(): + if not env.get('GITHUB_KEY', None) and not env.get('GITHUB_LOGIN', None): + log('WARN: Could not find "GITHUB_LOGIN" / "GITHUB_PASSWORD" or "GITHUB_KEY" in the env variables. You could need it.') + +def check_email_settings(): + if not env.get('MAIL_SENDER', None): + raise RuntimeError('Could not find "MAIL_SENDER"') + +# we print a notice if we can not find the relevant infos in the ~/.m2/settings.xml +print_sonatype_notice() + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Builds and publishes a Elasticsearch Plugin Release') + parser.add_argument('--branch', '-b', metavar='master', default=get_current_branch(), + help='The branch to release from. Defaults to the current branch.') + parser.add_argument('--skiptests', '-t', dest='tests', action='store_false', + help='Skips tests before release. Tests are run by default.') + parser.set_defaults(tests=True) + parser.add_argument('--remote', '-r', metavar='origin', default='origin', + help='The remote to push the release commit and tag to. Default is [origin]') + parser.add_argument('--publish', '-p', dest='dryrun', action='store_false', + help='Publishes the release. Disable by default.') + parser.add_argument('--disable_mail', '-dm', dest='mail', action='store_false', + help='Do not send a release email. Email is sent by default.') + + parser.set_defaults(dryrun=True) + parser.set_defaults(mail=True) + args = parser.parse_args() + + src_branch = args.branch + remote = args.remote + run_tests = args.tests + dry_run = args.dryrun + mail = args.mail + + if not dry_run: + check_s3_credentials() + print('WARNING: dryrun is set to "false" - this will push and publish the release') + if mail: + check_email_settings() + print('An email to %s will be sent after the release' + % env.get('MAIL_TO', 'elasticsearch@googlegroups.com')) + input('Press Enter to continue...') + + check_github_credentials() + + print(''.join(['-' for _ in range(80)])) + print('Preparing Release from branch [%s] running tests: [%s] dryrun: [%s]' % (src_branch, run_tests, dry_run)) + print(' JAVA_HOME is [%s]' % JAVA_HOME) + print(' Running with maven command: [%s] ' % (MVN)) + + release_version = find_release_version(src_branch) + artifact_id = find_from_pom('artifactId') + artifact_name = find_from_pom('name') + artifact_description = find_from_pom('description') + project_url = find_from_pom('url') + print(' Artifact Id: [%s]' % artifact_id) + print(' Release version: [%s]' % release_version) + + # extract snapshot + default_snapshot_version = guess_snapshot(release_version) + snapshot_version = input('Enter next snapshot version [%s]:' % default_snapshot_version) + snapshot_version = snapshot_version or default_snapshot_version + + print(' Next version: [%s-SNAPSHOT]' % snapshot_version) + print(' Artifact Name: [%s]' % artifact_name) + print(' Artifact Description: [%s]' % artifact_description) + print(' Project URL: [%s]' % project_url) + + if not dry_run: + smoke_test_version = release_version + head_hash = get_head_hash() + run_mvn('clean') # clean the env! + create_release_branch(remote, src_branch, release_version) + print(' Created release branch [%s]' % (release_branch(release_version))) + success = False + try: + pending_files = [POM_FILE, README_FILE] + remove_maven_snapshot(POM_FILE, release_version) + remove_version_snapshot(README_FILE, release_version) + set_date(README_FILE) + set_install_instructions(README_FILE, artifact_id, release_version) + print(' Done removing snapshot version') + add_pending_files(*pending_files) # expects var args use * to expand + commit_release(artifact_id, release_version) + print(' Committed release version [%s]' % release_version) + print(''.join(['-' for _ in range(80)])) + print('Building Release candidate') + input('Press Enter to continue...') + print(' Checking github issues') + repository = get_github_repository(artifact_id) + check_opened_issues(release_version, repository, artifact_id) + if not dry_run: + print(' Running maven builds now and publish to sonatype - run-tests [%s]' % run_tests) + else: + print(' Running maven builds now run-tests [%s]' % run_tests) + build_release(run_tests=run_tests, dry_run=dry_run) + artifact = get_artifacts(artifact_id, release_version) + artifact_and_checksums = generate_checksums(artifact) + print(''.join(['-' for _ in range(80)])) + + print('Finish Release -- dry_run: %s' % dry_run) + input('Press Enter to continue...') + print(' merge release branch') + git_merge(src_branch, release_version) + print(' tag') + tag_release(release_version) + + add_maven_snapshot(POM_FILE, release_version, snapshot_version) + add_version_snapshot(README_FILE, release_version, snapshot_version) + add_pending_files(*pending_files) + commit_snapshot() + + print(' push to %s %s -- dry_run: %s' % (remote, src_branch, dry_run)) + git_push(remote, src_branch, release_version, dry_run) + print(' publish artifacts to S3 -- dry_run: %s' % dry_run) + publish_artifacts(artifact_and_checksums, base='elasticsearch/%s' % (artifact_id) , dry_run=dry_run) + print(' preparing email (from github issues)') + msg = prepare_email(artifact_id, release_version, repository, artifact_name, artifact_description, project_url) + print(' sending email -- dry_run: %s, mail: %s' % (dry_run, mail)) + send_email(msg, dry_run=dry_run, mail=mail) + + pending_msg = """ +Release successful pending steps: + * close and release sonatype repo: https://oss.sonatype.org/ + * check if the release is there https://oss.sonatype.org/content/repositories/releases/org/elasticsearch/%(artifact_id)s/%(version)s + * tweet about the release +""" + print(pending_msg % {'version': release_version, + 'artifact_id': artifact_id, + 'project_url': project_url}) + success = True + finally: + if not success: + run('git reset --hard HEAD') + run('git checkout %s' % src_branch) + elif dry_run: + print('End of dry_run') + input('Press Enter to reset changes...') + + run('git reset --hard %s' % head_hash) + run('git tag -d v%s' % release_version) + # we delete this one anyways + run('git branch -D %s' % (release_branch(release_version))) diff --git a/dev-tools/upload-s3.py b/dev-tools/upload-s3.py new file mode 100644 index 00000000000..95ea576e65c --- /dev/null +++ b/dev-tools/upload-s3.py @@ -0,0 +1,67 @@ +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on +# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import os +import sys +import argparse +try: + import boto.s3 +except: + raise RuntimeError(""" + S3 upload requires boto to be installed + Use one of: + 'pip install -U boto' + 'apt-get install python-boto' + 'easy_install boto' + """) + +import boto.s3 + + +def list_buckets(conn): + return conn.get_all_buckets() + + +def upload_s3(conn, path, key, file, bucket): + print 'Uploading %s to Amazon S3 bucket %s/%s' % \ + (file, bucket, os.path.join(path, key)) + def percent_cb(complete, total): + sys.stdout.write('.') + sys.stdout.flush() + bucket = conn.create_bucket(bucket) + k = bucket.new_key(os.path.join(path, key)) + k.set_contents_from_filename(file, cb=percent_cb, num_cb=100) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Uploads files to Amazon S3') + parser.add_argument('--file', '-f', metavar='path to file', + help='the branch to release from', required=True) + parser.add_argument('--bucket', '-b', metavar='B42', default='download.elasticsearch.org', + help='The S3 Bucket to upload to') + parser.add_argument('--path', '-p', metavar='elasticsearch/elasticsearch', default='elasticsearch/elasticsearch', + help='The key path to use') + parser.add_argument('--key', '-k', metavar='key', default=None, + help='The key - uses the file name as default key') + args = parser.parse_args() + if args.key: + key = args.key + else: + key = os.path.basename(args.file) + + connection = boto.connect_s3() + upload_s3(connection, args.path, key, args.file, args.bucket); + diff --git a/pom.xml b/pom.xml index e4e0e8d17b0..8ab6eef18ff 100644 --- a/pom.xml +++ b/pom.xml @@ -2,13 +2,14 @@ - elasticsearch-lang-python 4.0.0 org.elasticsearch elasticsearch-lang-python 2.0.0-SNAPSHOT jar - JavaScript lang plugin for ElasticSearch + Elasticsearch Python language plugin + The Python language plugin allows to have python as the language of scripts to execute. + https://github.com/elasticsearch/elasticsearch-lang-python/ 2009 From 7b89ac3458257f2e3700d1acc0db3b7ad6ebbdee Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 7 Mar 2014 17:19:30 +0100 Subject: [PATCH 23/73] Add plugin version in es-plugin.properties With https://github.com/elasticsearch/elasticsearch/issues/2784, we can now add plugin version in `es-plugin.properties` file. It will only be used with elasticsearch 1.0.0 and upper. No need to push it in 1.x branch. Closes #9. --- pom.xml | 9 +++++++++ src/main/resources/es-plugin.properties | 1 + 2 files changed, 10 insertions(+) diff --git a/pom.xml b/pom.xml index 8ab6eef18ff..f7035f04ce0 100644 --- a/pom.xml +++ b/pom.xml @@ -100,6 +100,15 @@ + + + src/main/resources + true + + **/*.properties + + + org.apache.maven.plugins diff --git a/src/main/resources/es-plugin.properties b/src/main/resources/es-plugin.properties index a24df54c363..b0ed0d5e3e0 100644 --- a/src/main/resources/es-plugin.properties +++ b/src/main/resources/es-plugin.properties @@ -1 +1,2 @@ plugin=org.elasticsearch.plugin.python.PythonPlugin +version=${project.version} From 491e2e3958a0e80a15477b7f0beede49b4055c76 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 7 Mar 2014 17:23:26 +0100 Subject: [PATCH 24/73] Update to elasticsearch 1.0.0 Closes #10. --- pom.xml | 4 ++-- .../python/PythonScriptSearchTests.java | 20 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pom.xml b/pom.xml index f7035f04ce0..235bf06c691 100644 --- a/pom.xml +++ b/pom.xml @@ -31,8 +31,8 @@ - 1.0.0.RC1 - 4.6.0 + 1.0.0 + 4.6.1 1 true onerror diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java index 2f90291d651..9f2bfbb8477 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.script.python; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.hamcrest.CoreMatchers; @@ -115,18 +116,13 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { SearchResponse response = client().prepareSearch() .setQuery(matchAllQuery()) - .addField("_source.obj1") // we also automatically detect _source in fields .addScriptField("s_obj1", "python", "_source['obj1']", null) .addScriptField("s_obj1_test", "python", "_source['obj1']['test']", null) .addScriptField("s_obj2", "python", "_source['obj2']", null) .addScriptField("s_obj2_arr2", "python", "_source['obj2']['arr2']", null) .execute().actionGet(); - Map sObj1 = (Map) response.getHits().getAt(0).field("_source.obj1").value(); - assertThat(sObj1.get("test").toString(), equalTo("something")); - assertThat(response.getHits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something")); - - sObj1 = (Map) response.getHits().getAt(0).field("s_obj1").value(); + Map sObj1 = (Map) response.getHits().getAt(0).field("s_obj1").value(); assertThat(sObj1.get("test").toString(), equalTo("something")); assertThat(response.getHits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something")); @@ -155,7 +151,8 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { logger.info(" --> running doc['num1'].value"); SearchResponse response = client().search(searchRequest() .searchType(SearchType.QUERY_THEN_FETCH) - .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("doc['num1'].value").lang("python"))) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("doc['num1'].value").lang("python")))) ).actionGet(); assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); @@ -169,7 +166,8 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { logger.info(" --> running -doc['num1'].value"); response = client().search(searchRequest() .searchType(SearchType.QUERY_THEN_FETCH) - .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("-doc['num1'].value").lang("python"))) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("-doc['num1'].value").lang("python")))) ).actionGet(); assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); @@ -184,7 +182,8 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { logger.info(" --> running doc['num1'].value * _score"); response = client().search(searchRequest() .searchType(SearchType.QUERY_THEN_FETCH) - .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("doc['num1'].value * _score").lang("python"))) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("doc['num1'].value * _score").lang("python")))) ).actionGet(); assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); @@ -198,7 +197,8 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { logger.info(" --> running param1 * param2 * _score"); response = client().search(searchRequest() .searchType(SearchType.QUERY_THEN_FETCH) - .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("param1 * param2 * _score").param("param1", 2).param("param2", 2).lang("python"))) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("param1 * param2 * _score").param("param1", 2).param("param2", 2).lang("python")))) ).actionGet(); assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); From 97b63f230f755087719924e748d4ab39e84aa2b3 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 7 Mar 2014 17:34:04 +0100 Subject: [PATCH 25/73] Add documentation Closes #2. --- README.md | 124 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/README.md b/README.md index f07a6fe296f..f5612362720 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,130 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e | 2.0.0-SNAPSHOT | 1.0.0.RC1 -> master | 2.5.3 | XXXX-XX-XX | | 2.0.0.RC1 | 1.0.0.RC1 -> master | 2.5.3 | 2014-01-15 | +Using python with function_score +-------------------------------- + +Let's say you want to use `function_score` API using `python`. Here is +a way of doing it: + +```sh +curl -XDELETE "http://localhost:9200/test" + +curl -XPUT "http://localhost:9200/test/doc/1" -d '{ + "num": 1.0 +}' + +curl -XPUT "http://localhost:9200/test/doc/2?refresh" -d '{ + "num": 2.0 +}' + +curl -XGET "http://localhost:9200/test/_search?pretty" -d' +{ + "query": { + "function_score": { + "script_score": { + "script": "doc[\"num\"].value * _score", + "lang": "python" + } + } + } +}' +``` + +gives + +```javascript +{ + // ... + "hits": { + "total": 2, + "max_score": 2, + "hits": [ + { + // ... + "_score": 2 + }, + { + // ... + "_score": 1 + } + ] + } +} +``` + +Using python with script_fields +------------------------------- + +```sh +curl -XDELETE "http://localhost:9200/test" + +curl -XPUT "http://localhost:9200/test/doc/1?refresh" -d' +{ + "obj1": { + "test": "something" + }, + "obj2": { + "arr2": [ "arr_value1", "arr_value2" ] + } +}' + +curl -XGET "http://localhost:9200/test/_search" -d' +{ + "script_fields": { + "s_obj1": { + "script": "_source[\"obj1\"]", "lang": "python" + }, + "s_obj1_test": { + "script": "_source[\"obj1\"][\"test\"]", "lang": "python" + }, + "s_obj2": { + "script": "_source[\"obj2\"]", "lang": "python" + }, + "s_obj2_arr2": { + "script": "_source[\"obj2\"][\"arr2\"]", "lang": "python" + } + } +}' +``` + +gives + +```javascript +{ + // ... + "hits": [ + { + // ... + "fields": { + "s_obj2_arr2": [ + [ + "arr_value1", + "arr_value2" + ] + ], + "s_obj1_test": [ + "something" + ], + "s_obj2": [ + { + "arr2": [ + "arr_value1", + "arr_value2" + ] + } + ], + "s_obj1": [ + { + "test": "something" + } + ] + } + } + ] +} +``` + License ------- From 3303878d9a2993a22fe5ed50711f4f01f3c3b12a Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 7 Mar 2014 17:37:59 +0100 Subject: [PATCH 26/73] prepare release elasticsearch-lang-python-2.0.0 --- README.md | 4 ++-- pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f5612362720..c711c7f7386 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,14 @@ Python lang Plugin for Elasticsearch The Python (jython) language plugin allows to have `python` as the language of scripts to execute. -In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/2.0.0.RC1`. +In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/2.0.0`. * For 1.0.x elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/master). * For 0.90.x elasticsearch versions, look at [1.x branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/1.x). | Python Lang Plugin | elasticsearch | jython | Release date | |-----------------------------|---------------------|----------|:------------:| -| 2.0.0-SNAPSHOT | 1.0.0.RC1 -> master | 2.5.3 | XXXX-XX-XX | +| 2.0.0 | 1.0.0.RC1 -> master | 2.5.3 | 2014-03-07 | | 2.0.0.RC1 | 1.0.0.RC1 -> master | 2.5.3 | 2014-01-15 | Using python with function_score diff --git a/pom.xml b/pom.xml index 235bf06c691..3cd789f5425 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.elasticsearch elasticsearch-lang-python - 2.0.0-SNAPSHOT + 2.0.0 jar Elasticsearch Python language plugin The Python language plugin allows to have python as the language of scripts to execute. From 1b79c72e2d540b40d060cc2ce60b706b29c27deb Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 7 Mar 2014 17:41:37 +0100 Subject: [PATCH 27/73] prepare for next development iteration --- README.md | 1 + pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c711c7f7386..5d0824eb192 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e | Python Lang Plugin | elasticsearch | jython | Release date | |-----------------------------|---------------------|----------|:------------:| +| 2.1.0-SNAPSHOT | 1.0.0.RC1 -> master | 2.5.3 | XXXX-XX-XX | | 2.0.0 | 1.0.0.RC1 -> master | 2.5.3 | 2014-03-07 | | 2.0.0.RC1 | 1.0.0.RC1 -> master | 2.5.3 | 2014-01-15 | diff --git a/pom.xml b/pom.xml index 3cd789f5425..e0a87935db6 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.elasticsearch elasticsearch-lang-python - 2.0.0 + 2.1.0-SNAPSHOT jar Elasticsearch Python language plugin The Python language plugin allows to have python as the language of scripts to execute. From d76bdb12daade25287d5968cda7ca0a1c274536a Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 19 Mar 2014 22:42:50 +0100 Subject: [PATCH 28/73] Disable java and maven version checking And fix typo in email html --- dev-tools/build_release.py | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/dev-tools/build_release.py b/dev-tools/build_release.py index 9166b09e7e3..74acd8c5f4e 100755 --- a/dev-tools/build_release.py +++ b/dev-tools/build_release.py @@ -43,7 +43,7 @@ from os.path import dirname, abspath The script takes over almost all steps necessary for a release from a high level point of view it does the following things: - - run prerequisite checks ie. check for Java 1.6 being present or S3 credentials available as env variables + - run prerequisite checks ie. check for S3 credentials available as env variables - detect the version to release from the specified branch (--branch) or the current branch - creates a release branch & updates pom.xml and README.md to point to a release version rather than a snapshot - builds the artifacts @@ -109,18 +109,6 @@ def java_exe(): path = JAVA_HOME return 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % (path, path, path) -def verify_java_version(version): - s = os.popen('%s; java -version 2>&1' % java_exe()).read() - if s.find(' version "%s.' % version) == -1: - raise RuntimeError('got wrong version for java %s:\n%s' % (version, s)) - -# Verifies the java version. We guarantee that we run with Java 1.6 -# If 1.6 is not available fail the build! -def verify_mvn_java_version(version, mvn): - s = os.popen('%s; %s --version 2>&1' % (java_exe(), mvn)).read() - if s.find('Java version: %s' % version) == -1: - raise RuntimeError('got wrong java version for %s %s:\n%s' % (mvn, version, s)) - # Returns the hash of the current git HEAD revision def get_head_hash(): return os.popen(' git rev-parse --verify HEAD 2>&1').read().strip() @@ -133,9 +121,6 @@ def get_tag_hash(tag): def get_current_branch(): return os.popen('git rev-parse --abbrev-ref HEAD 2>&1').read().strip() -verify_java_version('1.6') # we require to build with 1.6 -verify_mvn_java_version('1.6', MVN) - # Utility that returns the name of the release branch for a given version def release_branch(version): return 'release_branch_%s' % version @@ -218,7 +203,7 @@ def add_version_snapshot(readme_file, release, snapshot): # If we find pattern, we copy the line and replace its content if line.find(pattern) >= 0: return line.replace(pattern, replacement).replace('%s' % (datetime.datetime.now().strftime("%Y-%m-%d")), - 'XXXX-XX-XX')+line + 'XXXX-XX-XX')+line else: return line process_file(readme_file, callback) @@ -356,15 +341,15 @@ def format_issues_html(issues, title='Fix'): if len(issues) > 0: response += '

%s

\n
    \n' % title for issue in issues: - response += '[%s] - %s\n' % (issue.html_url, issue.number, issue.title) + response += '
  • [%s] - %s\n' % (issue.html_url, issue.number, issue.title) response += '
\n' return response def get_github_repository(reponame, - login=env.get('GITHUB_LOGIN', None), - password=env.get('GITHUB_PASSWORD', None), - key=env.get('GITHUB_KEY', None)): + login=env.get('GITHUB_LOGIN', None), + password=env.get('GITHUB_PASSWORD', None), + key=env.get('GITHUB_KEY', None)): if login: g = github3.login(login, password) elif key: From 2c18d35b594727a4a93f354d80f6bf12849db14f Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 28 Mar 2014 18:35:25 +0100 Subject: [PATCH 29/73] Create branches according to elasticsearch versions We create branches: * es-0.90 for elasticsearch 0.90 * es-1.0 for elasticsearch 1.0 * es-1.1 for elasticsearch 1.1 * master for elasticsearch master We also check that before releasing we don't have a dependency to an elasticsearch SNAPSHOT version. Add links to each version in documentation --- README.md | 14 +++++---- dev-tools/build_release.py | 29 +++++++++++++++++++ pom.xml | 6 ++-- .../python/PythonScriptSearchTests.java | 4 --- 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 5d0824eb192..e296620891e 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,18 @@ The Python (jython) language plugin allows to have `python` as the language of s In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/2.0.0`. -* For 1.0.x elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/master). -* For 0.90.x elasticsearch versions, look at [1.x branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/1.x). +* For master elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/master). +* For 1.1.x elasticsearch versions, look at [es-1.1 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.1). +* For 1.0.x elasticsearch versions, look at [es-1.0 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.0). +* For 0.90.x elasticsearch versions, look at [es-0.90 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-0.90). | Python Lang Plugin | elasticsearch | jython | Release date | |-----------------------------|---------------------|----------|:------------:| -| 2.1.0-SNAPSHOT | 1.0.0.RC1 -> master | 2.5.3 | XXXX-XX-XX | -| 2.0.0 | 1.0.0.RC1 -> master | 2.5.3 | 2014-03-07 | -| 2.0.0.RC1 | 1.0.0.RC1 -> master | 2.5.3 | 2014-01-15 | +| 3.0.0-SNAPSHOT | master | 2.5.3 | XXXX-XX-XX | + +Please read documentation relative to the version you are using: + +* [3.0.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/blob/master/README.md) Using python with function_score -------------------------------- diff --git a/dev-tools/build_release.py b/dev-tools/build_release.py index 74acd8c5f4e..db8345440c7 100755 --- a/dev-tools/build_release.py +++ b/dev-tools/build_release.py @@ -208,6 +208,29 @@ def add_version_snapshot(readme_file, release, snapshot): return line process_file(readme_file, callback) +# Moves the README.md file from a snapshot to a release (documentation link) +def remove_documentation_snapshot(readme_file, repo_url, release, branch): + pattern = '* [%s-SNAPSHOT](%sblob/%s/README.md)' % (release, repo_url, branch) + replacement = '* [%s](%sblob/v%s/README.md)' % (release, repo_url, release) + def callback(line): + # If we find pattern, we replace its content + if line.find(pattern) >= 0: + return line.replace(pattern, replacement) + else: + return line + process_file(readme_file, callback) + +# Add in README.markdown file the documentation for the next version +def add_documentation_snapshot(readme_file, repo_url, release, snapshot, branch): + pattern = '* [%s](%sblob/v%s/README.md)' % (release, repo_url, release) + replacement = '* [%s-SNAPSHOT](%sblob/%s/README.md)' % (snapshot, repo_url, branch) + def callback(line): + # If we find pattern, we copy the line and replace its content + if line.find(pattern) >= 0: + return line.replace(pattern, replacement)+line + else: + return line + process_file(readme_file, callback) # Set release date in README.md file def set_date(readme_file): @@ -603,8 +626,12 @@ if __name__ == '__main__': artifact_name = find_from_pom('name') artifact_description = find_from_pom('description') project_url = find_from_pom('url') + elasticsearch_version = find_from_pom('elasticsearch.version') print(' Artifact Id: [%s]' % artifact_id) print(' Release version: [%s]' % release_version) + print(' Elasticsearch: [%s]' % elasticsearch_version) + if elasticsearch_version.find('-SNAPSHOT') != -1: + raise RuntimeError('Can not release with a SNAPSHOT elasticsearch dependency: %s' % elasticsearch_version) # extract snapshot default_snapshot_version = guess_snapshot(release_version) @@ -626,6 +653,7 @@ if __name__ == '__main__': try: pending_files = [POM_FILE, README_FILE] remove_maven_snapshot(POM_FILE, release_version) + remove_documentation_snapshot(README_FILE, project_url, release_version, src_branch) remove_version_snapshot(README_FILE, release_version) set_date(README_FILE) set_install_instructions(README_FILE, artifact_id, release_version) @@ -657,6 +685,7 @@ if __name__ == '__main__': add_maven_snapshot(POM_FILE, release_version, snapshot_version) add_version_snapshot(README_FILE, release_version, snapshot_version) + add_documentation_snapshot(README_FILE, project_url, release_version, snapshot_version, src_branch) add_pending_files(*pending_files) commit_snapshot() diff --git a/pom.xml b/pom.xml index e0a87935db6..f49cfecaf4f 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.elasticsearch elasticsearch-lang-python - 2.1.0-SNAPSHOT + 3.0.0-SNAPSHOT jar Elasticsearch Python language plugin The Python language plugin allows to have python as the language of scripts to execute. @@ -31,8 +31,8 @@ - 1.0.0 - 4.6.1 + 2.0.0-SNAPSHOT + 4.7.0 1 true onerror diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java index 9f2bfbb8477..d05a1d6b0e9 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java @@ -54,7 +54,6 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { @Test public void testPythonFilter() throws Exception { - wipeIndices("test"); createIndex("test"); index("test", "type1", "1", jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject()); flush(); @@ -105,7 +104,6 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { @Test public void testScriptFieldUsingSource() throws Exception { - wipeIndices("test"); createIndex("test"); index("test", "type1", "1", jsonBuilder().startObject() @@ -140,7 +138,6 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { @Test public void testCustomScriptBoost() throws Exception { - wipeIndices("test"); createIndex("test"); index("test", "type1", "1", jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject()); index("test", "type1", "2", jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject()); @@ -215,7 +212,6 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { */ @Test public void testPythonEmptyParameters() throws Exception { - wipeIndices("test"); createIndex("test"); index("test", "type1", "1", jsonBuilder().startObject().field("myfield", "foo").endObject()); refresh(); From b651570dde5b185df2d070f5d581764d199b03de Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 23 May 2014 17:20:48 +0200 Subject: [PATCH 30/73] Update to elasticsearch 1.2.0 Closes #11. (cherry picked from commit 8a87054) --- README.md | 1 + pom.xml | 6 +++--- .../script/python/PythonScriptEngineService.java | 5 +++++ .../script/python/PythonScriptMultiThreadedTest.java | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e296620891e..b56a67864ea 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ The Python (jython) language plugin allows to have `python` as the language of s In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/2.0.0`. * For master elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/master). +* For 1.2.x elasticsearch versions, look at [es-1.2 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.2). * For 1.1.x elasticsearch versions, look at [es-1.1 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.1). * For 1.0.x elasticsearch versions, look at [es-1.0 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.0). * For 0.90.x elasticsearch versions, look at [es-0.90 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-0.90). diff --git a/pom.xml b/pom.xml index f49cfecaf4f..8f2882e4a4b 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ 2.0.0-SNAPSHOT - 4.7.0 + 4.8.1 1 true onerror @@ -115,8 +115,8 @@ maven-compiler-plugin 2.3.2 - 1.6 - 1.6 + 1.7 + 1.7 diff --git a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java index 2a94949830d..14d22fdecb9 100644 --- a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java @@ -62,6 +62,11 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri return new String[]{"py"}; } + @Override + public boolean sandboxed() { + return false; + } + @Override public Object compile(String script) { return interp.compile(script); diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java b/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java index 0af0505eeca..0cd49cd21fe 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java @@ -20,7 +20,6 @@ package org.elasticsearch.script.python; import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.util.concurrent.jsr166y.ThreadLocalRandom; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.test.ElasticsearchTestCase; import org.junit.After; @@ -30,6 +29,7 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicBoolean; import static org.hamcrest.Matchers.equalTo; From 9492736ab70fff6eacff518468ee9e5f13857418 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 23 May 2014 17:47:11 +0200 Subject: [PATCH 31/73] Update to elasticsearch 1.3.0 Closes #13. (cherry picked from commit b9cff46) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b56a67864ea..c9685499508 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ The Python (jython) language plugin allows to have `python` as the language of s In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/2.0.0`. * For master elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/master). +* For 1.3.x elasticsearch versions, look at [es-1.3 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.3). * For 1.2.x elasticsearch versions, look at [es-1.2 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.2). * For 1.1.x elasticsearch versions, look at [es-1.1 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.1). * For 1.0.x elasticsearch versions, look at [es-1.0 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.0). From 1e46301d8346e02bada754641fbf03249b45751d Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 23 May 2014 20:35:14 +0200 Subject: [PATCH 32/73] [TEST] fix tests Tests fail because we now disable automatic plugin loading from the classpat when running tests (cherry picked from commit c5554ad) --- .../script/python/PythonScriptSearchTests.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java index d05a1d6b0e9..bbd4c385e4a 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java @@ -21,7 +21,10 @@ package org.elasticsearch.script.python; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.hamcrest.CoreMatchers; @@ -43,8 +46,17 @@ import static org.hamcrest.Matchers.equalTo; /** * */ +@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.SUITE) public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return ImmutableSettings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .build(); + } + @After public void close() { // We need to clear some system properties From c58e49e84a5f965d03f3f4bbaa8212ab444eb8c4 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 23 Jul 2014 23:50:16 +0200 Subject: [PATCH 33/73] Update to Lucene 4.9.0 / elasticsearch 1.3.0 Closes #15. Related to #13. (cherry picked from commit 8f077d6) --- pom.xml | 2 +- .../elasticsearch/script/python/PythonScriptSearchTests.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 8f2882e4a4b..0c34e69e906 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ 2.0.0-SNAPSHOT - 4.8.1 + 4.9.0 1 true onerror diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java index bbd4c385e4a..b24bc067e4b 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.hamcrest.CoreMatchers; @@ -228,7 +229,8 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { index("test", "type1", "1", jsonBuilder().startObject().field("myfield", "foo").endObject()); refresh(); - client().prepareUpdate("test", "type1", "1").setScriptLang("python").setScript("ctx[\"_source\"][\"myfield\"]=\"bar\"") + client().prepareUpdate("test", "type1", "1").setScriptLang("python") + .setScript("ctx[\"_source\"][\"myfield\"]=\"bar\"", ScriptService.ScriptType.INLINE) .execute().actionGet(); refresh(); From 870e8b8aa2349b19cb419bf64555d56f9c8567d1 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 23 Jul 2014 23:51:38 +0200 Subject: [PATCH 34/73] Update to elasticsearch 1.4.0 Closes #14. (cherry picked from commit 8d677aa) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c9685499508..76ef41a5cfd 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ The Python (jython) language plugin allows to have `python` as the language of s In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/2.0.0`. * For master elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/master). +* For 1.4.x elasticsearch versions, look at [es-1.4 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.4). * For 1.3.x elasticsearch versions, look at [es-1.3 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.3). * For 1.2.x elasticsearch versions, look at [es-1.2 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.2). * For 1.1.x elasticsearch versions, look at [es-1.1 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.1). From d985f6b0496bf6607784d7856ed5d4e08b987a9a Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 24 Jul 2014 00:15:23 +0200 Subject: [PATCH 35/73] Update to elasticsearch 1.4.0 Related to #14. (cherry picked from commit 964de0d) --- .../script/python/PythonScriptEngineService.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java index 14d22fdecb9..d71d3fb0066 100644 --- a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java @@ -182,11 +182,6 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri lookup.source().setNextSource(source); } - @Override - public void setNextScore(float score) { - pyVars.__setitem__("_score", Py.java2py(score)); - } - @Override public void setNextVar(String name, Object value) { pyVars.__setitem__(name, Py.java2py(value)); From 2bfac65f74ac87df1f05a3d5a4a1b05c271f9c6c Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 8 Sep 2014 23:57:57 +0200 Subject: [PATCH 36/73] Update to Lucene 4.10.0 Closes #20. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0c34e69e906..0691d70816a 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ 2.0.0-SNAPSHOT - 4.9.0 + 4.10.0 1 true onerror From c1b4018f87b54c819dde8384211d23d14d41d885 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 9 Sep 2014 00:02:33 +0200 Subject: [PATCH 37/73] Docs: make the welcome page more obvious Closes #17. --- .gitignore | 1 + README.md | 36 +- dev-tools/build_release.py | 722 ------------------------------------- dev-tools/release.py | 134 +++++++ dev-tools/upload-s3.py | 67 ---- 5 files changed, 158 insertions(+), 802 deletions(-) delete mode 100755 dev-tools/build_release.py create mode 100644 dev-tools/release.py delete mode 100644 dev-tools/upload-s3.py diff --git a/.gitignore b/.gitignore index 21d2106f97c..cfc70c0c989 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ .DS_Store *.iml .local-execution-hints.log +/plugin_tools diff --git a/README.md b/README.md index 76ef41a5cfd..e33ecb1d7cc 100644 --- a/README.md +++ b/README.md @@ -3,23 +3,33 @@ Python lang Plugin for Elasticsearch The Python (jython) language plugin allows to have `python` as the language of scripts to execute. -In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-lang-python/2.0.0`. +In order to install the plugin, simply run: -* For master elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/master). -* For 1.4.x elasticsearch versions, look at [es-1.4 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.4). -* For 1.3.x elasticsearch versions, look at [es-1.3 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.3). -* For 1.2.x elasticsearch versions, look at [es-1.2 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.2). -* For 1.1.x elasticsearch versions, look at [es-1.1 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.1). -* For 1.0.x elasticsearch versions, look at [es-1.0 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.0). -* For 0.90.x elasticsearch versions, look at [es-0.90 branch](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-0.90). +```sh +bin/plugin -install elasticsearch/elasticsearch-lang-python/2.3.0 +``` -| Python Lang Plugin | elasticsearch | jython | Release date | -|-----------------------------|---------------------|----------|:------------:| -| 3.0.0-SNAPSHOT | master | 2.5.3 | XXXX-XX-XX | +You need to install a version matching your Elasticsearch version: -Please read documentation relative to the version you are using: +| elasticsearch | Python Lang Plugin | Docs | +|---------------|-----------------------|------------| +| master | Build from source | See below | +| es-1.x | Build from source | [2.4.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.x/#version-240-snapshot-for-elasticsearch-1x) | +| es-1.3 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.0/#python-lang-plugin-for-elasticsearch) | +| es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.2.0/#python-lang-plugin-for-elasticsearch) | +| es-1.0 | 2.0.0 | [2.0.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.0.0/#python-lang-plugin-for-elasticsearch) | +| es-0.90 | 1.0.0 | [1.0.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v1.0.0/#python-lang-plugin-for-elasticsearch) | -* [3.0.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/blob/master/README.md) +To build a `SNAPSHOT` version, you need to build it with Maven: + +```bash +mvn clean install +plugin --install lang-python \ + --url file:target/releases/elasticsearch-lang-python-X.X.X-SNAPSHOT.zip +``` + +User Guide +---------- Using python with function_score -------------------------------- diff --git a/dev-tools/build_release.py b/dev-tools/build_release.py deleted file mode 100755 index db8345440c7..00000000000 --- a/dev-tools/build_release.py +++ /dev/null @@ -1,722 +0,0 @@ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on -# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - -import re -import tempfile -import shutil -import os -import datetime -import argparse -import github3 -import smtplib - -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText - -from os.path import dirname, abspath - -""" - This tool builds a release from the a given elasticsearch plugin branch. - In order to execute it go in the top level directory and run: - $ python3 dev_tools/build_release.py --branch master --publish --remote origin - - By default this script runs in 'dry' mode which essentially simulates a release. If the - '--publish' option is set the actual release is done. - If not in 'dry' mode, a mail will be automatically sent to the mailing list. - You can disable it with the option '--disable_mail' - - $ python3 dev_tools/build_release.py --publish --remote origin --disable_mail - - The script takes over almost all - steps necessary for a release from a high level point of view it does the following things: - - - run prerequisite checks ie. check for S3 credentials available as env variables - - detect the version to release from the specified branch (--branch) or the current branch - - creates a release branch & updates pom.xml and README.md to point to a release version rather than a snapshot - - builds the artifacts - - commits the new version and merges the release branch into the source branch - - creates a tag and pushes the commit to the specified origin (--remote) - - publishes the releases to sonatype and S3 - - send a mail based on github issues fixed by this version - -Once it's done it will print all the remaining steps. - - Prerequisites: - - Python 3k for script execution - - Boto for S3 Upload ($ apt-get install python-boto or pip-3.3 install boto) - - github3 module (pip-3.3 install github3.py) - - S3 keys exported via ENV Variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) - - GITHUB (login/password) or key exported via ENV Variables (GITHUB_LOGIN, GITHUB_PASSWORD or GITHUB_KEY) - (see https://github.com/settings/applications#personal-access-tokens) - Optional: default to no authentication - - SMTP_HOST - Optional: default to localhost - - MAIL_SENDER - Optional: default to 'david@pilato.fr': must be authorized to send emails to elasticsearch mailing list - - MAIL_TO - Optional: default to 'elasticsearch@googlegroups.com' -""" -env = os.environ - -LOG = env.get('ES_RELEASE_LOG', '/tmp/elasticsearch_release.log') -ROOT_DIR = os.path.join(abspath(dirname(__file__)), '../') -README_FILE = ROOT_DIR + 'README.md' -POM_FILE = ROOT_DIR + 'pom.xml' - -def log(msg): - log_plain('\n%s' % msg) - -def log_plain(msg): - f = open(LOG, mode='ab') - f.write(msg.encode('utf-8')) - f.close() - -def run(command, quiet=False): - log('%s: RUN: %s\n' % (datetime.datetime.now(), command)) - if os.system('%s >> %s 2>&1' % (command, LOG)): - msg = ' FAILED: %s [see log %s]' % (command, LOG) - if not quiet: - print(msg) - raise RuntimeError(msg) - -try: - JAVA_HOME = env['JAVA_HOME'] -except KeyError: - raise RuntimeError(""" - Please set JAVA_HOME in the env before running release tool - On OSX use: export JAVA_HOME=`/usr/libexec/java_home -v '1.6*'`""") - -try: - MVN='mvn' - # make sure mvn3 is used if mvn3 is available - # some systems use maven 2 as default - run('mvn3 --version', quiet=True) - MVN='mvn3' -except RuntimeError: - pass - - -def java_exe(): - path = JAVA_HOME - return 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % (path, path, path) - -# Returns the hash of the current git HEAD revision -def get_head_hash(): - return os.popen(' git rev-parse --verify HEAD 2>&1').read().strip() - -# Returns the hash of the given tag revision -def get_tag_hash(tag): - return os.popen('git show-ref --tags %s --hash 2>&1' % (tag)).read().strip() - -# Returns the name of the current branch -def get_current_branch(): - return os.popen('git rev-parse --abbrev-ref HEAD 2>&1').read().strip() - -# Utility that returns the name of the release branch for a given version -def release_branch(version): - return 'release_branch_%s' % version - -# runs get fetch on the given remote -def fetch(remote): - run('git fetch %s' % remote) - -# Creates a new release branch from the given source branch -# and rebases the source branch from the remote before creating -# the release branch. Note: This fails if the source branch -# doesn't exist on the provided remote. -def create_release_branch(remote, src_branch, release): - run('git checkout %s' % src_branch) - run('git pull --rebase %s %s' % (remote, src_branch)) - run('git checkout -b %s' % (release_branch(release))) - - -# Reads the given file and applies the -# callback to it. If the callback changed -# a line the given file is replaced with -# the modified input. -def process_file(file_path, line_callback): - fh, abs_path = tempfile.mkstemp() - modified = False - with open(abs_path,'w', encoding='utf-8') as new_file: - with open(file_path, encoding='utf-8') as old_file: - for line in old_file: - new_line = line_callback(line) - modified = modified or (new_line != line) - new_file.write(new_line) - os.close(fh) - if modified: - #Remove original file - os.remove(file_path) - #Move new file - shutil.move(abs_path, file_path) - return True - else: - # nothing to do - just remove the tmp file - os.remove(abs_path) - return False - -# Guess the next snapshot version number (increment second digit) -def guess_snapshot(version): - digits=list(map(int, re.findall(r'\d+', version))) - source='%s.%s' % (digits[0], digits[1]) - destination='%s.%s' % (digits[0], digits[1]+1) - return version.replace(source, destination) - -# Moves the pom.xml file from a snapshot to a release -def remove_maven_snapshot(pom, release): - pattern = '%s-SNAPSHOT' % release - replacement = '%s' % release - def callback(line): - return line.replace(pattern, replacement) - process_file(pom, callback) - -# Moves the README.md file from a snapshot to a release -def remove_version_snapshot(readme_file, release): - pattern = '%s-SNAPSHOT' % release - replacement = '%s ' % release - def callback(line): - return line.replace(pattern, replacement) - process_file(readme_file, callback) - -# Moves the pom.xml file to the next snapshot -def add_maven_snapshot(pom, release, snapshot): - pattern = '%s' % release - replacement = '%s-SNAPSHOT' % snapshot - def callback(line): - return line.replace(pattern, replacement) - process_file(pom, callback) - -# Add in README.md file the next snapshot -def add_version_snapshot(readme_file, release, snapshot): - pattern = '| %s ' % release - replacement = '| %s-SNAPSHOT' % snapshot - def callback(line): - # If we find pattern, we copy the line and replace its content - if line.find(pattern) >= 0: - return line.replace(pattern, replacement).replace('%s' % (datetime.datetime.now().strftime("%Y-%m-%d")), - 'XXXX-XX-XX')+line - else: - return line - process_file(readme_file, callback) - -# Moves the README.md file from a snapshot to a release (documentation link) -def remove_documentation_snapshot(readme_file, repo_url, release, branch): - pattern = '* [%s-SNAPSHOT](%sblob/%s/README.md)' % (release, repo_url, branch) - replacement = '* [%s](%sblob/v%s/README.md)' % (release, repo_url, release) - def callback(line): - # If we find pattern, we replace its content - if line.find(pattern) >= 0: - return line.replace(pattern, replacement) - else: - return line - process_file(readme_file, callback) - -# Add in README.markdown file the documentation for the next version -def add_documentation_snapshot(readme_file, repo_url, release, snapshot, branch): - pattern = '* [%s](%sblob/v%s/README.md)' % (release, repo_url, release) - replacement = '* [%s-SNAPSHOT](%sblob/%s/README.md)' % (snapshot, repo_url, branch) - def callback(line): - # If we find pattern, we copy the line and replace its content - if line.find(pattern) >= 0: - return line.replace(pattern, replacement)+line - else: - return line - process_file(readme_file, callback) - -# Set release date in README.md file -def set_date(readme_file): - pattern = 'XXXX-XX-XX' - replacement = '%s' % (datetime.datetime.now().strftime("%Y-%m-%d")) - def callback(line): - return line.replace(pattern, replacement) - process_file(readme_file, callback) - -# Update installation instructions in README.md file -def set_install_instructions(readme_file, artifact_name, release): - pattern = '`bin/plugin -install elasticsearch/%s/.+`' % artifact_name - replacement = '`bin/plugin -install elasticsearch/%s/%s`' % (artifact_name, release) - def callback(line): - return re.sub(pattern, replacement, line) - process_file(readme_file, callback) - - -# Stages the given files for the next git commit -def add_pending_files(*files): - for file in files: - run('git add %s' % file) - -# Executes a git commit with 'release [version]' as the commit message -def commit_release(artifact_id, release): - run('git commit -m "prepare release %s-%s"' % (artifact_id, release)) - -def commit_snapshot(): - run('git commit -m "prepare for next development iteration"') - -def tag_release(release): - run('git tag -a v%s -m "Tag release version %s"' % (release, release)) - -def run_mvn(*cmd): - for c in cmd: - run('%s; %s -f %s %s' % (java_exe(), MVN, POM_FILE, c)) - -def build_release(run_tests=False, dry_run=True): - target = 'deploy' - if dry_run: - target = 'package' - if run_tests: - run_mvn('clean test') - run_mvn('clean %s -DskipTests' %(target)) - -# Checks the pom.xml for the release version. 2.0.0-SNAPSHOT -# This method fails if the pom file has no SNAPSHOT version set ie. -# if the version is already on a release version we fail. -# Returns the next version string ie. 0.90.7 -def find_release_version(src_branch): - run('git checkout %s' % src_branch) - with open(POM_FILE, encoding='utf-8') as file: - for line in file: - match = re.search(r'(.+)-SNAPSHOT', line) - if match: - return match.group(1) - raise RuntimeError('Could not find release version in branch %s' % src_branch) - -# extract a value from pom.xml -def find_from_pom(tag): - with open(POM_FILE, encoding='utf-8') as file: - for line in file: - match = re.search(r'<%s>(.+)' % (tag, tag), line) - if match: - return match.group(1) - raise RuntimeError('Could not find <%s> in pom.xml file' % (tag)) - -def get_artifacts(artifact_id, release): - artifact_path = ROOT_DIR + 'target/releases/%s-%s.zip' % (artifact_id, release) - print(' Path %s' % (artifact_path)) - if not os.path.isfile(artifact_path): - raise RuntimeError('Could not find required artifact at %s' % (artifact_path)) - return artifact_path - -# Generates sha1 for a file -# and returns the checksum files as well -# as the given files in a list -def generate_checksums(release_file): - res = [] - directory = os.path.dirname(release_file) - file = os.path.basename(release_file) - checksum_file = '%s.sha1.txt' % file - - if os.system('cd %s; shasum %s > %s' % (directory, file, checksum_file)): - raise RuntimeError('Failed to generate checksum for file %s' % release_file) - res = res + [os.path.join(directory, checksum_file), release_file] - return res - -def git_merge(src_branch, release_version): - run('git checkout %s' % src_branch) - run('git merge %s' % release_branch(release_version)) - -def git_push(remote, src_branch, release_version, dry_run): - if not dry_run: - run('git push %s %s' % (remote, src_branch)) # push the commit - run('git push %s v%s' % (remote, release_version)) # push the tag - else: - print(' dryrun [True] -- skipping push to remote %s' % remote) - -def publish_artifacts(artifacts, base='elasticsearch/elasticsearch', dry_run=True): - location = os.path.dirname(os.path.realpath(__file__)) - for artifact in artifacts: - if dry_run: - print('Skip Uploading %s to Amazon S3 in %s' % (artifact, base)) - else: - print('Uploading %s to Amazon S3' % artifact) - # requires boto to be installed but it is not available on python3k yet so we use a dedicated tool - run('python %s/upload-s3.py --file %s --path %s' % (location, os.path.abspath(artifact), base)) - - -################# -## -## -## Email and Github Management -## -## -################# -def format_issues_plain(issues, title='Fix'): - response = "" - - if len(issues) > 0: - response += '%s:\n' % title - for issue in issues: - response += ' * [%s] - %s (%s)\n' % (issue.number, issue.title, issue.html_url) - - return response - -def format_issues_html(issues, title='Fix'): - response = "" - - if len(issues) > 0: - response += '

%s

\n
    \n' % title - for issue in issues: - response += '
  • [%s] - %s\n' % (issue.html_url, issue.number, issue.title) - response += '
\n' - - return response - -def get_github_repository(reponame, - login=env.get('GITHUB_LOGIN', None), - password=env.get('GITHUB_PASSWORD', None), - key=env.get('GITHUB_KEY', None)): - if login: - g = github3.login(login, password) - elif key: - g = github3.login(token=key) - else: - g = github3.GitHub() - - return g.repository("elasticsearch", reponame) - -# Check if there are some remaining open issues and fails -def check_opened_issues(version, repository, reponame): - opened_issues = [i for i in repository.iter_issues(state='open', labels='%s' % version)] - if len(opened_issues)>0: - raise NameError('Some issues [%s] are still opened. Check https://github.com/elasticsearch/%s/issues?labels=%s&state=open' - % (len(opened_issues), reponame, version)) - -# List issues from github: can be done anonymously if you don't -# exceed a given number of github API calls per day -# Check if there are some remaining open issues and fails -def list_issues(version, - repository, - severity='bug'): - issues = [i for i in repository.iter_issues(state='closed', labels='%s,%s' % (severity, version))] - return issues - -# Get issues from github and generates a Plain/HTML Multipart email -# And send it if dry_run=False -def prepare_email(artifact_id, release_version, repository, - artifact_name, artifact_description, project_url, - severity_labels_bug='bug', - severity_labels_update='update', - severity_labels_new='new', - severity_labels_doc='doc'): - - ## Get bugs from github - issues_bug = list_issues(release_version, repository, severity=severity_labels_bug) - issues_update = list_issues(release_version, repository, severity=severity_labels_update) - issues_new = list_issues(release_version, repository, severity=severity_labels_new) - issues_doc = list_issues(release_version, repository, severity=severity_labels_doc) - - ## Format content to plain text - plain_issues_bug = format_issues_plain(issues_bug, 'Fix') - plain_issues_update = format_issues_plain(issues_update, 'Update') - plain_issues_new = format_issues_plain(issues_new, 'New') - plain_issues_doc = format_issues_plain(issues_doc, 'Doc') - - ## Format content to html - html_issues_bug = format_issues_html(issues_bug, 'Fix') - html_issues_update = format_issues_html(issues_update, 'Update') - html_issues_new = format_issues_html(issues_new, 'New') - html_issues_doc = format_issues_html(issues_doc, 'Doc') - - if len(issues_bug)+len(issues_update)+len(issues_new)+len(issues_doc) > 0: - plain_empty_message = "" - html_empty_message = "" - - else: - plain_empty_message = "No issue listed for this release" - html_empty_message = "

No issue listed for this release

" - - msg = MIMEMultipart('alternative') - msg['Subject'] = '[ANN] %s %s released' % (artifact_name, release_version) - text = """ -Heya, - - -We are pleased to announce the release of the %(artifact_name)s, version %(release_version)s. - -%(artifact_description)s. - -%(project_url)s - -Release Notes - %(artifact_id)s - Version %(release_version)s - -%(empty_message)s -%(issues_bug)s -%(issues_update)s -%(issues_new)s -%(issues_doc)s - -Issues, Pull requests, Feature requests are warmly welcome on %(artifact_id)s project repository: %(project_url)s -For questions or comments around this plugin, feel free to use elasticsearch mailing list: https://groups.google.com/forum/#!forum/elasticsearch - -Enjoy, - --The Elasticsearch team -""" % {'release_version': release_version, - 'artifact_id': artifact_id, - 'artifact_name': artifact_name, - 'artifact_description': artifact_description, - 'project_url': project_url, - 'empty_message': plain_empty_message, - 'issues_bug': plain_issues_bug, - 'issues_update': plain_issues_update, - 'issues_new': plain_issues_new, - 'issues_doc': plain_issues_doc} - - html = """ - - -

Heya,

- -

We are pleased to announce the release of the %(artifact_name)s, version %(release_version)s

- -
%(artifact_description)s.
- -

Release Notes - Version %(release_version)s

-%(empty_message)s -%(issues_bug)s -%(issues_update)s -%(issues_new)s -%(issues_doc)s - -

Issues, Pull requests, Feature requests are warmly welcome on -%(artifact_id)s project repository!

-

For questions or comments around this plugin, feel free to use elasticsearch -mailing list!

- -

Enjoy,

- -

- The Elasticsearch team

- -""" % {'release_version': release_version, - 'artifact_id': artifact_id, - 'artifact_name': artifact_name, - 'artifact_description': artifact_description, - 'project_url': project_url, - 'empty_message': html_empty_message, - 'issues_bug': html_issues_bug, - 'issues_update': html_issues_update, - 'issues_new': html_issues_new, - 'issues_doc': html_issues_doc} - - # Record the MIME types of both parts - text/plain and text/html. - part1 = MIMEText(text, 'plain') - part2 = MIMEText(html, 'html') - - # Attach parts into message container. - # According to RFC 2046, the last part of a multipart message, in this case - # the HTML message, is best and preferred. - msg.attach(part1) - msg.attach(part2) - - return msg - -def send_email(msg, - dry_run=True, - mail=True, - sender=env.get('MAIL_SENDER'), - to=env.get('MAIL_TO', 'elasticsearch@googlegroups.com'), - smtp_server=env.get('SMTP_SERVER', 'localhost')): - msg['From'] = 'Elasticsearch Team <%s>' % sender - msg['To'] = 'Elasticsearch Mailing List <%s>' % to - # save mail on disk - with open(ROOT_DIR+'target/email.txt', 'w') as email_file: - email_file.write(msg.as_string()) - if mail and not dry_run: - s = smtplib.SMTP(smtp_server, 25) - s.sendmail(sender, to, msg.as_string()) - s.quit() - else: - print('generated email: open %starget/email.txt' % ROOT_DIR) - -def print_sonatype_notice(): - settings = os.path.join(os.path.expanduser('~'), '.m2/settings.xml') - if os.path.isfile(settings): - with open(settings, encoding='utf-8') as settings_file: - for line in settings_file: - if line.strip() == 'sonatype-nexus-snapshots': - # moving out - we found the indicator no need to print the warning - return - print(""" - NOTE: No sonatype settings detected, make sure you have configured - your sonatype credentials in '~/.m2/settings.xml': - - - ... - - - sonatype-nexus-snapshots - your-jira-id - your-jira-pwd - - - sonatype-nexus-staging - your-jira-id - your-jira-pwd - - - ... - - """) - -def check_s3_credentials(): - if not env.get('AWS_ACCESS_KEY_ID', None) or not env.get('AWS_SECRET_ACCESS_KEY', None): - raise RuntimeError('Could not find "AWS_ACCESS_KEY_ID" / "AWS_SECRET_ACCESS_KEY" in the env variables please export in order to upload to S3') - -def check_github_credentials(): - if not env.get('GITHUB_KEY', None) and not env.get('GITHUB_LOGIN', None): - log('WARN: Could not find "GITHUB_LOGIN" / "GITHUB_PASSWORD" or "GITHUB_KEY" in the env variables. You could need it.') - -def check_email_settings(): - if not env.get('MAIL_SENDER', None): - raise RuntimeError('Could not find "MAIL_SENDER"') - -# we print a notice if we can not find the relevant infos in the ~/.m2/settings.xml -print_sonatype_notice() - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Builds and publishes a Elasticsearch Plugin Release') - parser.add_argument('--branch', '-b', metavar='master', default=get_current_branch(), - help='The branch to release from. Defaults to the current branch.') - parser.add_argument('--skiptests', '-t', dest='tests', action='store_false', - help='Skips tests before release. Tests are run by default.') - parser.set_defaults(tests=True) - parser.add_argument('--remote', '-r', metavar='origin', default='origin', - help='The remote to push the release commit and tag to. Default is [origin]') - parser.add_argument('--publish', '-p', dest='dryrun', action='store_false', - help='Publishes the release. Disable by default.') - parser.add_argument('--disable_mail', '-dm', dest='mail', action='store_false', - help='Do not send a release email. Email is sent by default.') - - parser.set_defaults(dryrun=True) - parser.set_defaults(mail=True) - args = parser.parse_args() - - src_branch = args.branch - remote = args.remote - run_tests = args.tests - dry_run = args.dryrun - mail = args.mail - - if not dry_run: - check_s3_credentials() - print('WARNING: dryrun is set to "false" - this will push and publish the release') - if mail: - check_email_settings() - print('An email to %s will be sent after the release' - % env.get('MAIL_TO', 'elasticsearch@googlegroups.com')) - input('Press Enter to continue...') - - check_github_credentials() - - print(''.join(['-' for _ in range(80)])) - print('Preparing Release from branch [%s] running tests: [%s] dryrun: [%s]' % (src_branch, run_tests, dry_run)) - print(' JAVA_HOME is [%s]' % JAVA_HOME) - print(' Running with maven command: [%s] ' % (MVN)) - - release_version = find_release_version(src_branch) - artifact_id = find_from_pom('artifactId') - artifact_name = find_from_pom('name') - artifact_description = find_from_pom('description') - project_url = find_from_pom('url') - elasticsearch_version = find_from_pom('elasticsearch.version') - print(' Artifact Id: [%s]' % artifact_id) - print(' Release version: [%s]' % release_version) - print(' Elasticsearch: [%s]' % elasticsearch_version) - if elasticsearch_version.find('-SNAPSHOT') != -1: - raise RuntimeError('Can not release with a SNAPSHOT elasticsearch dependency: %s' % elasticsearch_version) - - # extract snapshot - default_snapshot_version = guess_snapshot(release_version) - snapshot_version = input('Enter next snapshot version [%s]:' % default_snapshot_version) - snapshot_version = snapshot_version or default_snapshot_version - - print(' Next version: [%s-SNAPSHOT]' % snapshot_version) - print(' Artifact Name: [%s]' % artifact_name) - print(' Artifact Description: [%s]' % artifact_description) - print(' Project URL: [%s]' % project_url) - - if not dry_run: - smoke_test_version = release_version - head_hash = get_head_hash() - run_mvn('clean') # clean the env! - create_release_branch(remote, src_branch, release_version) - print(' Created release branch [%s]' % (release_branch(release_version))) - success = False - try: - pending_files = [POM_FILE, README_FILE] - remove_maven_snapshot(POM_FILE, release_version) - remove_documentation_snapshot(README_FILE, project_url, release_version, src_branch) - remove_version_snapshot(README_FILE, release_version) - set_date(README_FILE) - set_install_instructions(README_FILE, artifact_id, release_version) - print(' Done removing snapshot version') - add_pending_files(*pending_files) # expects var args use * to expand - commit_release(artifact_id, release_version) - print(' Committed release version [%s]' % release_version) - print(''.join(['-' for _ in range(80)])) - print('Building Release candidate') - input('Press Enter to continue...') - print(' Checking github issues') - repository = get_github_repository(artifact_id) - check_opened_issues(release_version, repository, artifact_id) - if not dry_run: - print(' Running maven builds now and publish to sonatype - run-tests [%s]' % run_tests) - else: - print(' Running maven builds now run-tests [%s]' % run_tests) - build_release(run_tests=run_tests, dry_run=dry_run) - artifact = get_artifacts(artifact_id, release_version) - artifact_and_checksums = generate_checksums(artifact) - print(''.join(['-' for _ in range(80)])) - - print('Finish Release -- dry_run: %s' % dry_run) - input('Press Enter to continue...') - print(' merge release branch') - git_merge(src_branch, release_version) - print(' tag') - tag_release(release_version) - - add_maven_snapshot(POM_FILE, release_version, snapshot_version) - add_version_snapshot(README_FILE, release_version, snapshot_version) - add_documentation_snapshot(README_FILE, project_url, release_version, snapshot_version, src_branch) - add_pending_files(*pending_files) - commit_snapshot() - - print(' push to %s %s -- dry_run: %s' % (remote, src_branch, dry_run)) - git_push(remote, src_branch, release_version, dry_run) - print(' publish artifacts to S3 -- dry_run: %s' % dry_run) - publish_artifacts(artifact_and_checksums, base='elasticsearch/%s' % (artifact_id) , dry_run=dry_run) - print(' preparing email (from github issues)') - msg = prepare_email(artifact_id, release_version, repository, artifact_name, artifact_description, project_url) - print(' sending email -- dry_run: %s, mail: %s' % (dry_run, mail)) - send_email(msg, dry_run=dry_run, mail=mail) - - pending_msg = """ -Release successful pending steps: - * close and release sonatype repo: https://oss.sonatype.org/ - * check if the release is there https://oss.sonatype.org/content/repositories/releases/org/elasticsearch/%(artifact_id)s/%(version)s - * tweet about the release -""" - print(pending_msg % {'version': release_version, - 'artifact_id': artifact_id, - 'project_url': project_url}) - success = True - finally: - if not success: - run('git reset --hard HEAD') - run('git checkout %s' % src_branch) - elif dry_run: - print('End of dry_run') - input('Press Enter to reset changes...') - - run('git reset --hard %s' % head_hash) - run('git tag -d v%s' % release_version) - # we delete this one anyways - run('git branch -D %s' % (release_branch(release_version))) diff --git a/dev-tools/release.py b/dev-tools/release.py new file mode 100644 index 00000000000..edcc637d068 --- /dev/null +++ b/dev-tools/release.py @@ -0,0 +1,134 @@ +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on +# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import datetime +import os +import shutil +import sys +import time +import urllib +import urllib.request +import zipfile + +from os.path import dirname, abspath + +""" + This tool builds a release from the a given elasticsearch plugin branch. + + It is basically a wrapper on top of launch_release.py which: + + - tries to get a more recent version of launch_release.py in ... + - download it if needed + - launch it passing all arguments to it, like: + + $ python3 dev_tools/release.py --branch master --publish --remote origin + + Important options: + + # Dry run + $ python3 dev_tools/release.py + + # Dry run without tests + python3 dev_tools/release.py --skiptests + + # Release, publish artifacts and announce + $ python3 dev_tools/release.py --publish + + See full documentation in launch_release.py +""" +env = os.environ + +# Change this if the source repository for your scripts is at a different location +SOURCE_REPO = 'elasticsearch/elasticsearch-plugins-script' +# We define that we should download again the script after 1 days +SCRIPT_OBSOLETE_DAYS = 1 +# We ignore in master.zip file the following files +IGNORED_FILES = ['.gitignore', 'README.md'] + + +ROOT_DIR = abspath(os.path.join(abspath(dirname(__file__)), '../')) +TARGET_TOOLS_DIR = ROOT_DIR + '/plugin_tools' +DEV_TOOLS_DIR = ROOT_DIR + '/dev-tools' +BUILD_RELEASE_FILENAME = 'release.zip' +BUILD_RELEASE_FILE = TARGET_TOOLS_DIR + '/' + BUILD_RELEASE_FILENAME +SOURCE_URL = 'https://github.com/%s/archive/master.zip' % SOURCE_REPO + +# Download a recent version of the release plugin tool +try: + os.mkdir(TARGET_TOOLS_DIR) + print('directory %s created' % TARGET_TOOLS_DIR) +except FileExistsError: + pass + + +try: + # we check latest update. If we ran an update recently, we + # are not going to check it again + download = True + + try: + last_download_time = datetime.datetime.fromtimestamp(os.path.getmtime(BUILD_RELEASE_FILE)) + if (datetime.datetime.now()-last_download_time).days < SCRIPT_OBSOLETE_DAYS: + download = False + except FileNotFoundError: + pass + + if download: + urllib.request.urlretrieve(SOURCE_URL, BUILD_RELEASE_FILE) + with zipfile.ZipFile(BUILD_RELEASE_FILE) as myzip: + for member in myzip.infolist(): + filename = os.path.basename(member.filename) + # skip directories + if not filename: + continue + if filename in IGNORED_FILES: + continue + + # copy file (taken from zipfile's extract) + source = myzip.open(member.filename) + target = open(os.path.join(TARGET_TOOLS_DIR, filename), "wb") + with source, target: + shutil.copyfileobj(source, target) + # We keep the original date + date_time = time.mktime(member.date_time + (0, 0, -1)) + os.utime(os.path.join(TARGET_TOOLS_DIR, filename), (date_time, date_time)) + print('plugin-tools updated from %s' % SOURCE_URL) +except urllib.error.HTTPError: + pass + + +# Let see if we need to update the release.py script itself +source_time = os.path.getmtime(TARGET_TOOLS_DIR + '/release.py') +repo_time = os.path.getmtime(DEV_TOOLS_DIR + '/release.py') +if source_time > repo_time: + input('release.py needs an update. Press a key to update it...') + shutil.copyfile(TARGET_TOOLS_DIR + '/release.py', DEV_TOOLS_DIR + '/release.py') + +# We can launch the build process +try: + PYTHON = 'python' + # make sure python3 is used if python3 is available + # some systems use python 2 as default + os.system('python3 --version > /dev/null 2>&1') + PYTHON = 'python3' +except RuntimeError: + pass + +release_args = '' +for x in range(1, len(sys.argv)): + release_args += ' ' + sys.argv[x] + +os.system('%s %s/build_release.py %s' % (PYTHON, TARGET_TOOLS_DIR, release_args)) diff --git a/dev-tools/upload-s3.py b/dev-tools/upload-s3.py deleted file mode 100644 index 95ea576e65c..00000000000 --- a/dev-tools/upload-s3.py +++ /dev/null @@ -1,67 +0,0 @@ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on -# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - -import os -import sys -import argparse -try: - import boto.s3 -except: - raise RuntimeError(""" - S3 upload requires boto to be installed - Use one of: - 'pip install -U boto' - 'apt-get install python-boto' - 'easy_install boto' - """) - -import boto.s3 - - -def list_buckets(conn): - return conn.get_all_buckets() - - -def upload_s3(conn, path, key, file, bucket): - print 'Uploading %s to Amazon S3 bucket %s/%s' % \ - (file, bucket, os.path.join(path, key)) - def percent_cb(complete, total): - sys.stdout.write('.') - sys.stdout.flush() - bucket = conn.create_bucket(bucket) - k = bucket.new_key(os.path.join(path, key)) - k.set_contents_from_filename(file, cb=percent_cb, num_cb=100) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Uploads files to Amazon S3') - parser.add_argument('--file', '-f', metavar='path to file', - help='the branch to release from', required=True) - parser.add_argument('--bucket', '-b', metavar='B42', default='download.elasticsearch.org', - help='The S3 Bucket to upload to') - parser.add_argument('--path', '-p', metavar='elasticsearch/elasticsearch', default='elasticsearch/elasticsearch', - help='The key path to use') - parser.add_argument('--key', '-k', metavar='key', default=None, - help='The key - uses the file name as default key') - args = parser.parse_args() - if args.key: - key = args.key - else: - key = os.path.basename(args.file) - - connection = boto.connect_s3() - upload_s3(connection, args.path, key, args.file, args.bucket); - From 4fd60126809f22ab1d93b9839d98b407f960ad81 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 15 Sep 2014 10:42:31 +0200 Subject: [PATCH 38/73] Create branch es-1.4 for elasticsearch 1.4.0 --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e33ecb1d7cc..256bb0c635b 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ You need to install a version matching your Elasticsearch version: | elasticsearch | Python Lang Plugin | Docs | |---------------|-----------------------|------------| | master | Build from source | See below | -| es-1.x | Build from source | [2.4.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.x/#version-240-snapshot-for-elasticsearch-1x) | +| es-1.x | Build from source | [2.5.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.x/#version-250-snapshot-for-elasticsearch-1x) | +| es-1.4 | Build from source | [2.4.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.4/#version-240-snapshot-for-elasticsearch-1x) | | es-1.3 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.0/#python-lang-plugin-for-elasticsearch) | | es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.2.0/#python-lang-plugin-for-elasticsearch) | | es-1.0 | 2.0.0 | [2.0.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.0.0/#python-lang-plugin-for-elasticsearch) | From da5f0f785e90bec2461b17ab7295c064e93e80c1 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Wed, 24 Sep 2014 18:10:55 -0400 Subject: [PATCH 39/73] Upgrade to Lucene 4.10.1 snapshot --- pom.xml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 0691d70816a..2f7b31b8074 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,8 @@ 2.0.0-SNAPSHOT - 4.10.0 + 4.10.1 + 4.10.1-snapshot-1627368 1 true onerror @@ -41,6 +42,10 @@ + + Lucene snapshots + https://download.elasticsearch.org/lucenesnapshots/ + sonatype http://oss.sonatype.org/content/repositories/releases/ @@ -51,7 +56,7 @@ org.apache.lucene lucene-test-framework - ${lucene.version} + ${lucene.maven.version} test From f7b4a4de9cde2261bb862e0d2d4516c0b1778d7b Mon Sep 17 00:00:00 2001 From: mikemccand Date: Sun, 28 Sep 2014 17:57:50 -0400 Subject: [PATCH 40/73] Upgrade to Lucene 4.10.1 --- pom.xml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 2f7b31b8074..9384bba1c62 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,7 @@ 2.0.0-SNAPSHOT 4.10.1 - 4.10.1-snapshot-1627368 + 4.10.1 1 true onerror @@ -42,10 +42,6 @@ - - Lucene snapshots - https://download.elasticsearch.org/lucenesnapshots/ - sonatype http://oss.sonatype.org/content/repositories/releases/ From cd7756c283b4ddbe5cd561b60ecaaaf3c69ab0e2 Mon Sep 17 00:00:00 2001 From: Britta Weber Date: Tue, 30 Sep 2014 20:04:14 +0200 Subject: [PATCH 41/73] Remove setNextScore in SearchScript Due to a change in elasticsearch 1.4.0, we need to apply a similar patch here. See elasticsearch/elasticsearch#6864 See elasticsearch/elasticsearch#7819 Closes #16. Closes #21. --- .../python/PythonScriptEngineService.java | 3 +- .../python/PythonScriptSearchTests.java | 53 ++++++++++++++++++- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java index d71d3fb0066..8e48c4d5018 100644 --- a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.ScoreAccessor; import org.elasticsearch.script.ScriptEngineService; import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.lookup.SearchLookup; @@ -164,7 +165,7 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri @Override public void setScorer(Scorer scorer) { - lookup.setScorer(scorer); + pyVars.__setitem__("_score", Py.java2py(new ScoreAccessor(scorer))); } @Override diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java index b24bc067e4b..24778c99a36 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java @@ -26,12 +26,15 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.hamcrest.CoreMatchers; +import org.hamcrest.Matchers; import org.junit.After; import org.junit.Test; +import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -40,7 +43,10 @@ import static org.elasticsearch.client.Requests.searchRequest; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.FilterBuilders.scriptFilter; import static org.elasticsearch.index.query.QueryBuilders.*; +import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; +import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.equalTo; @@ -193,7 +199,7 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { response = client().search(searchRequest() .searchType(SearchType.QUERY_THEN_FETCH) .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) - .add(ScoreFunctionBuilders.scriptFunction("doc['num1'].value * _score").lang("python")))) + .add(ScoreFunctionBuilders.scriptFunction("doc['num1'].value * _score.doubleValue()").lang("python")))) ).actionGet(); assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); @@ -208,7 +214,7 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { response = client().search(searchRequest() .searchType(SearchType.QUERY_THEN_FETCH) .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) - .add(ScoreFunctionBuilders.scriptFunction("param1 * param2 * _score").param("param1", 2).param("param2", 2).lang("python")))) + .add(ScoreFunctionBuilders.scriptFunction("param1 * param2 * _score.doubleValue()").param("param1", 2).param("param2", 2).lang("python")))) ).actionGet(); assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); @@ -239,4 +245,47 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { assertThat((String) value, CoreMatchers.equalTo("bar")); } + + @Test + public void testScriptScoresNested() throws IOException { + createIndex("index"); + ensureYellow(); + index("index", "testtype", "1", jsonBuilder().startObject().field("dummy_field", 1).endObject()); + refresh(); + SearchResponse response = client().search( + searchRequest().source( + searchSource().query( + functionScoreQuery( + functionScoreQuery( + functionScoreQuery().add(scriptFunction("1").lang("python"))) + .add(scriptFunction("_score.doubleValue()").lang("python"))) + .add(scriptFunction("_score.doubleValue()").lang("python") + ) + ) + ) + ).actionGet(); + assertSearchResponse(response); + assertThat(response.getHits().getAt(0).score(), equalTo(1.0f)); + } + + @Test + public void testScriptScoresWithAgg() throws IOException { + createIndex("index"); + ensureYellow(); + index("index", "testtype", "1", jsonBuilder().startObject().field("dummy_field", 1).endObject()); + refresh(); + SearchResponse response = client().search( + searchRequest().source( + searchSource().query( + functionScoreQuery() + .add(scriptFunction("_score.doubleValue()").lang("python") + ) + ).aggregation(terms("score_agg").script("_score.doubleValue()").lang("python")) + ) + ).actionGet(); + assertSearchResponse(response); + assertThat(response.getHits().getAt(0).score(), equalTo(1.0f)); + assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getKeyAsNumber().floatValue(), Matchers.is(1f)); + assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getDocCount(), Matchers.is(1l)); + } } From bc8c065977870c69f03fa8af40c753822fc136c0 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 7 Oct 2014 13:13:50 +0200 Subject: [PATCH 42/73] Update to Jython 2.7-b3 It sounds like Jython 2.5.3 is leaking some threads. Jython 2.5.4.rc1 has the same issue. Jython 2.7-b3 fixes it. Typical error when running tests: ``` ERROR 0.00s J2 | PythonScriptEngineTests (suite) <<< > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.elasticsearch.script.python.PythonScriptEngineTests: > 1) Thread[id=12, name=org.python.google.common.base.internal.Finalizer, state=WAITING, group=TGRP-PythonScriptEngineTests] > at java.lang.Object.wait(Native Method) > at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) > at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) > at org.python.google.common.base.internal.Finalizer.run(Finalizer.java:127) > at __randomizedtesting.SeedInfo.seed([7A5ECFD8D0474383]:0) > Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: > 1) Thread[id=12, name=org.python.google.common.base.internal.Finalizer, state=WAITING, group=TGRP-PythonScriptEngineTests] > at java.lang.Object.wait(Native Method) > at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) > at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) > at org.python.google.common.base.internal.Finalizer.run(Finalizer.java:127) > at __randomizedtesting.SeedInfo.seed([7A5ECFD8D0474383]:0) ``` Closes #22. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9384bba1c62..1b7ad880f7f 100644 --- a/pom.xml +++ b/pom.xml @@ -66,7 +66,7 @@ org.python jython-standalone - 2.5.3 + 2.7-b3 compile From dfeeee6e79fc702633192e9c23dfe182decc74c7 Mon Sep 17 00:00:00 2001 From: Britta Weber Date: Tue, 7 Oct 2014 13:24:44 +0200 Subject: [PATCH 43/73] Multi-line or multi-statement Python scripts raise NullPointerException Calling a multi-statement score using curl e.g. `a=22; _score*a` is not possible with the current lang-python, nor is it possible to write a script using semicolons or multiple lines. I can only get compound statements (e.g. list comprehensions with embedded if statements) to work, so I'm limited in the complexity of my scoring process. I'm using ES 1.3.2 and latest lang-python: ``` $ ls -la /usr/share/elasticsearch/plugins/lang-python/ -rw-r--r-- 1 root root 10482 Aug 27 17:20 elasticsearch-lang-python-2.3.0.jar -rw-r--r-- 1 root root 14340135 Aug 27 17:20 jython-standalone-2.5.3.jar ``` Here's a worked example: ``` # Delete existing data, add 2 simple records, fetch both results to stdout curl -XDELETE "http://localhost:9200/test" curl -XPUT "http://localhost:9200/test/doc/1" -d '{ "num": 1.0 }' curl -XPUT "http://localhost:9200/test/doc/2?refresh" -d '{ "num": 2.0 }' # show our records curl -XGET 'http://localhost:9200/test/_search' -d '{ "query" : { "match_all": {}} }' ``` We'll run a simple query that uses `num` as a score modifier: ```doc["num"].value * _score``` If I create `/etc/elasticsearch/scripts/py1.py`: ``` doc["num"].value * _score ``` and wait for the script to reload (by monitoring the logs), I can call: ``` $ curl -XGET "http://localhost:9200/test/_search?pretty" -d' { "query": { "function_score": { "script_score": { "script": "py1", "lang": "python" } } } }' ``` and this will calculate the results. The same can be achieved using an in-line call (ignoring `py1.py`): ``` curl -XGET "http://localhost:9200/test/_search?pretty" -d' { "query": { "function_score": { "script_score": { "script": "doc[\"num\"].value * _score", "lang": "python" } } } }' ``` However using more than 1 statement will fail. This example uses `;` to split 2 statements (this is legal in jython 2.5): ``` curl -XGET "http://localhost:9200/test/_search?pretty" -d' { "query": { "function_score": { "script_score": { "script": "a=42; doc[\"num\"].value * _score", "lang": "python" } } } }' "reason" : "QueryPhaseExecutionException[[test][3]: query[function score (ConstantScore(*:*),function=script[a=42; doc[\"num\"].value * _score], params [null])],from[0],size[10]: Query Failed [Failed to execute main query]]; nested: NullPointerException; " and the log message: org.elasticsearch.search.query.QueryPhaseExecutionException: [test][3]: query[function score (ConstantScore(*:*),function=script[a=42; doc["num"].value * _score], params [null])],from[0],size[10]: Query Failed [Failed to execute main query] at org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:162) at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:261) at org.elasticsearch.search.action.SearchServiceTransportAction$5.call(SearchServiceTransportAction.java:206) at org.elasticsearch.search.action.SearchServiceTransportAction$5.call(SearchServiceTransportAction.java:203) at org.elasticsearch.search.action.SearchServiceTransportAction$23.run(SearchServiceTransportAction.java:517) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.NullPointerException ``` Creating a `py2.py` in the scripts directory containing: ``` a=42; doc["num"].value * _score ``` and calling ``` $ curl -XGET "http://localhost:9200/test/_search?pretty" -d' { "query": { "function_score": { "script_score": { "script": "py2", "lang": "python" } } } }' has the same error: "reason" : "QueryPhaseExecutionException[[test][3]: query[function score (ConstantScore(*:*),function=script[py2], params [null])],from[0],size[10]: Query Failed [Failed to execute main query]]; nested: PyException; " ``` If a `py3.py` script is made with the same two statements split over two lines: ``` a=42 doc["num"].value * _score ``` then the same errors are thrown. I'll note that if I experiment with equivalent groovy scripts then both the semicolon is allowed and multi-line scripts (in /scripts) are allowed. Closes #19. (cherry picked from commit 9fca562) --- .../python/PythonScriptSearchTests.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java index 24778c99a36..ad16ef05cb5 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java @@ -288,4 +288,26 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getKeyAsNumber().floatValue(), Matchers.is(1f)); assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getDocCount(), Matchers.is(1l)); } + + /** + * Test case for #19: https://github.com/elasticsearch/elasticsearch-lang-python/issues/19 + * Multi-line or multi-statement Python scripts raise NullPointerException + */ + @Test + public void testPythonMultiLines() throws Exception { + createIndex("test"); + index("test", "type1", "1", jsonBuilder().startObject().field("myfield", "foo").endObject()); + refresh(); + + client().prepareUpdate("test", "type1", "1").setScriptLang("python") + .setScript("a=42; ctx[\"_source\"][\"myfield\"]=\"bar\"", ScriptService.ScriptType.INLINE) + .execute().actionGet(); + refresh(); + + Object value = get("test", "type1", "1").getSourceAsMap().get("myfield"); + assertThat(value instanceof String, is(true)); + + assertThat((String) value, CoreMatchers.equalTo("bar")); + } + } From 336729021a1c8bdbd0f42403682910512dfdf0a4 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 7 Oct 2014 13:45:37 +0200 Subject: [PATCH 44/73] update documentation with release 2.4.0 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 256bb0c635b..5d923647fe8 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The Python (jython) language plugin allows to have `python` as the language of s In order to install the plugin, simply run: ```sh -bin/plugin -install elasticsearch/elasticsearch-lang-python/2.3.0 +bin/plugin -install elasticsearch/elasticsearch-lang-python/2.4.0 ``` You need to install a version matching your Elasticsearch version: @@ -15,7 +15,7 @@ You need to install a version matching your Elasticsearch version: |---------------|-----------------------|------------| | master | Build from source | See below | | es-1.x | Build from source | [2.5.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.x/#version-250-snapshot-for-elasticsearch-1x) | -| es-1.4 | Build from source | [2.4.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.4/#version-240-snapshot-for-elasticsearch-1x) | +| es-1.4 | 2.4.0 | [2.4.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.4.0/#version-240-for-elasticsearch-14) | | es-1.3 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.0/#python-lang-plugin-for-elasticsearch) | | es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.2.0/#python-lang-plugin-for-elasticsearch) | | es-1.0 | 2.0.0 | [2.0.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.0.0/#python-lang-plugin-for-elasticsearch) | From 8afd16885a3582f3678f57ae5bb8b0d37cf28892 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 15 Oct 2014 18:21:57 +0200 Subject: [PATCH 45/73] Implement new method `scriptRemoved(CompiledScript)` in `ScriptEngineService` This [PR](https://github.com/elasticsearch/elasticsearch/pull/8062) broke ScriptEngineService by adding a new method `scriptRemoved(CompiledScript)`. Closes #23. --- .../script/python/PythonScriptEngineService.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java index 8e48c4d5018..165463d9833 100644 --- a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java @@ -25,10 +25,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.ScoreAccessor; -import org.elasticsearch.script.ScriptEngineService; -import org.elasticsearch.script.SearchScript; +import org.elasticsearch.script.*; import org.elasticsearch.search.lookup.SearchLookup; import org.python.core.Py; import org.python.core.PyCode; @@ -104,6 +101,11 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri interp.cleanup(); } + @Override + public void scriptRemoved(@Nullable CompiledScript compiledScript) { + // Nothing to do + } + public class PythonExecutableScript implements ExecutableScript { private final PyCode code; From 12805b56b73f7db1509b7f3c5be64e33140cd609 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Thu, 30 Oct 2014 05:55:59 -0400 Subject: [PATCH 46/73] Upgrade to Lucene 4.10.2 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1b7ad880f7f..ae5f74d3b1f 100644 --- a/pom.xml +++ b/pom.xml @@ -32,8 +32,8 @@ 2.0.0-SNAPSHOT - 4.10.1 - 4.10.1 + 4.10.2 + 4.10.2 1 true onerror From 032dc51e9f613c6bda514934e44587fcab1d1fef Mon Sep 17 00:00:00 2001 From: tlrx Date: Mon, 3 Nov 2014 12:09:24 +0100 Subject: [PATCH 47/73] [TESTS] Upgrade randomizedtesting-runner to 2.1.10 Closes #24 --- pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pom.xml b/pom.xml index ae5f74d3b1f..d10965d19b8 100644 --- a/pom.xml +++ b/pom.xml @@ -98,6 +98,14 @@ 1.3.RC2 test + + + com.carrotsearch.randomizedtesting + randomizedtesting-runner + 2.1.10 + test + + From c7a66cc8abf20f1a8d185553eca68529b8b076b1 Mon Sep 17 00:00:00 2001 From: tlrx Date: Wed, 5 Nov 2014 17:57:36 +0100 Subject: [PATCH 48/73] update documentation with release 2.4.1 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5d923647fe8..e57a99d3317 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The Python (jython) language plugin allows to have `python` as the language of s In order to install the plugin, simply run: ```sh -bin/plugin -install elasticsearch/elasticsearch-lang-python/2.4.0 +bin/plugin -install elasticsearch/elasticsearch-lang-python/2.4.1 ``` You need to install a version matching your Elasticsearch version: @@ -15,7 +15,7 @@ You need to install a version matching your Elasticsearch version: |---------------|-----------------------|------------| | master | Build from source | See below | | es-1.x | Build from source | [2.5.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.x/#version-250-snapshot-for-elasticsearch-1x) | -| es-1.4 | 2.4.0 | [2.4.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.4.0/#version-240-for-elasticsearch-14) | +| es-1.4 | 2.4.1 | [2.4.1](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.4.1/#version-241-for-elasticsearch-14) | | es-1.3 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.0/#python-lang-plugin-for-elasticsearch) | | es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.2.0/#python-lang-plugin-for-elasticsearch) | | es-1.0 | 2.0.0 | [2.0.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.0.0/#python-lang-plugin-for-elasticsearch) | From 8cbdf5302f7caebe0d168fd9e5c4212d8d21bb89 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 5 Nov 2014 17:18:37 -0500 Subject: [PATCH 49/73] upgrade to lucene 5 snapshot --- pom.xml | 8 ++++++-- .../script/python/PythonScriptEngineService.java | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index d10965d19b8..0533b382636 100644 --- a/pom.xml +++ b/pom.xml @@ -32,8 +32,8 @@ 2.0.0-SNAPSHOT - 4.10.2 - 4.10.2 + 5.0.0 + 5.0.0-snapshot-1636426 1 true onerror @@ -46,6 +46,10 @@ sonatype http://oss.sonatype.org/content/repositories/releases/ + + Lucene snapshots + https://download.elasticsearch.org/lucenesnapshots/maven/ + diff --git a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java index 165463d9833..54e1543011b 100644 --- a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java @@ -19,7 +19,7 @@ package org.elasticsearch.script.python; -import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Scorer; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; @@ -171,7 +171,7 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri } @Override - public void setNextReader(AtomicReaderContext context) { + public void setNextReader(LeafReaderContext context) { lookup.setNextReader(context); } From 2f4861905dfcdbecb880ae2eb022fefbfb51a913 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Mon, 10 Nov 2014 16:46:08 -0500 Subject: [PATCH 50/73] Upgrade to Lucene 5.0.0-snapshot-1637347 --- pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 0533b382636..3ee4be9990c 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,7 @@ 2.0.0-SNAPSHOT 5.0.0 - 5.0.0-snapshot-1636426 + 5.0.0-snapshot-1637347 1 true onerror @@ -43,12 +43,12 @@ - sonatype - http://oss.sonatype.org/content/repositories/releases/ + Lucene snapshots + https://download.elasticsearch.org/lucenesnapshots/1637347/ - Lucene snapshots - https://download.elasticsearch.org/lucenesnapshots/maven/ + sonatype + http://oss.sonatype.org/content/repositories/releases/ From 99b08b0a5c8769aa796851cfdbcf27c65855fa14 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Mon, 24 Nov 2014 05:52:02 -0500 Subject: [PATCH 51/73] Upgrade to Lucene 5.0.0-snapshot-1641343 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 3ee4be9990c..e36880cbb44 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,7 @@ 2.0.0-SNAPSHOT 5.0.0 - 5.0.0-snapshot-1637347 + 5.0.0-snapshot-1641343 1 true onerror @@ -44,7 +44,7 @@ Lucene snapshots - https://download.elasticsearch.org/lucenesnapshots/1637347/ + https://download.elasticsearch.org/lucenesnapshots/1641343/ sonatype From 480a862be1b00f7c1b1937a3908bf4d7147b6d0b Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 2 Dec 2014 18:18:39 +0100 Subject: [PATCH 52/73] Upgrade to Lucene 5.0.0-snapshot-1642891 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index e36880cbb44..e5dc907538d 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,7 @@ 2.0.0-SNAPSHOT 5.0.0 - 5.0.0-snapshot-1641343 + 5.0.0-snapshot-1642891 1 true onerror @@ -44,7 +44,7 @@ Lucene snapshots - https://download.elasticsearch.org/lucenesnapshots/1641343/ + https://download.elasticsearch.org/lucenesnapshots/1642891/ sonatype From ea41c9fc160b8195c548df6b48815dcc200d27af Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 4 Dec 2014 09:48:45 +0100 Subject: [PATCH 53/73] [Test] replace field.value() to field.values() for multivalued fields --- .../elasticsearch/script/python/PythonScriptSearchTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java index ad16ef05cb5..3543bea7654 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java @@ -149,7 +149,7 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1")); assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2")); - sObj2Arr2 = (List) response.getHits().getAt(0).field("s_obj2_arr2").value(); + sObj2Arr2 = (List) response.getHits().getAt(0).field("s_obj2_arr2").values(); assertThat(sObj2Arr2.size(), equalTo(2)); assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1")); assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2")); From 1ec8d69c6e8416c80dcc3aeab9e38a95bb57aff1 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Sun, 14 Dec 2014 19:43:51 +0100 Subject: [PATCH 54/73] Depend on elasticsearch-parent To simplify plugins maintenance and provide more value in the future, we are starting to build an `elasticsearch-parent` project. This commit is the first step for this plugin to depend on this new `pom` maven project. --- .gitignore | 6 +- dev-tools/tests.policy | 54 +++++++++++++ pom.xml | 174 +++++------------------------------------ 3 files changed, 77 insertions(+), 157 deletions(-) create mode 100644 dev-tools/tests.policy diff --git a/.gitignore b/.gitignore index cfc70c0c989..9533848e238 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,9 @@ /target .DS_Store *.iml -.local-execution-hints.log +/.project +/.settings +/.classpath /plugin_tools +/.local-execution-hints.log +/.local-*-execution-hints.log diff --git a/dev-tools/tests.policy b/dev-tools/tests.policy new file mode 100644 index 00000000000..6afb5025840 --- /dev/null +++ b/dev-tools/tests.policy @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Policy file to prevent tests from writing outside the test sandbox directory +// PLEASE NOTE: You may need to enable other permissions when new tests are added, +// everything not allowed here is forbidden! + +grant { + // permissions for file access, write access only to sandbox: + permission java.io.FilePermission "<>", "read,execute"; + permission java.io.FilePermission "${junit4.childvm.cwd}", "read,execute,write"; + permission java.io.FilePermission "${junit4.childvm.cwd}${/}-", "read,execute,write,delete"; + permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,execute,write,delete"; + permission groovy.security.GroovyCodeSourcePermission "/groovy/script"; + + // Allow connecting to the internet anywhere + permission java.net.SocketPermission "*", "accept,listen,connect,resolve"; + + // Basic permissions needed for Lucene / Elasticsearch to work: + permission java.util.PropertyPermission "*", "read,write"; + permission java.lang.reflect.ReflectPermission "*"; + permission java.lang.RuntimePermission "*"; + + // These two *have* to be spelled out a separate + permission java.lang.management.ManagementPermission "control"; + permission java.lang.management.ManagementPermission "monitor"; + + permission java.net.NetPermission "*"; + permission java.util.logging.LoggingPermission "control"; + permission javax.management.MBeanPermission "*", "*"; + permission javax.management.MBeanServerPermission "*"; + permission javax.management.MBeanTrustPermission "*"; + + // Needed for some things in DNS caching in the JVM + permission java.security.SecurityPermission "getProperty.networkaddress.cache.ttl"; + permission java.security.SecurityPermission "getProperty.networkaddress.cache.negative.ttl"; + +}; diff --git a/pom.xml b/pom.xml index e5dc907538d..7ccad2ed947 100644 --- a/pom.xml +++ b/pom.xml @@ -3,6 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + + + org.elasticsearch + elasticsearch-parent + 2.0.0-SNAPSHOT + + org.elasticsearch elasticsearch-lang-python 3.0.0-SNAPSHOT @@ -24,92 +31,44 @@ http://github.com/elasticsearch/elasticsearch-lang-python - - org.sonatype.oss - oss-parent - 7 - - - 2.0.0-SNAPSHOT - 5.0.0 - 5.0.0-snapshot-1642891 - 1 - true - onerror - - INFO + - - - Lucene snapshots - https://download.elasticsearch.org/lucenesnapshots/1642891/ - - - sonatype - http://oss.sonatype.org/content/repositories/releases/ - - - + + org.hamcrest + hamcrest-all + + + com.carrotsearch.randomizedtesting + randomizedtesting-runner + org.apache.lucene lucene-test-framework - ${lucene.maven.version} - test - org.elasticsearch elasticsearch - ${elasticsearch.version} - compile + org.python jython-standalone 2.7-b3 - compile log4j log4j - 1.2.16 - runtime - org.elasticsearch elasticsearch - ${elasticsearch.version} test-jar - test - - - org.hamcrest - hamcrest-core - 1.3.RC2 - test - - - - org.hamcrest - hamcrest-library - 1.3.RC2 - test - - - - com.carrotsearch.randomizedtesting - randomizedtesting-runner - 2.1.10 - test - - @@ -126,120 +85,23 @@ org.apache.maven.plugins maven-compiler-plugin - 2.3.2 - - 1.7 - 1.7 - com.carrotsearch.randomizedtesting junit4-maven-plugin - 2.0.12 - - - tests - test - - junit4 - - - 20 - pipe,warn - true - - - - - - - - - ${tests.jvms} - - - - - - - **/*Tests.class - **/*Test.class - - - **/Abstract*.class - **/*StressTest.class - - - -Xmx512m - -XX:MaxDirectMemorySize=512m - -Des.logger.prefix= - - ${tests.shuffle} - ${tests.verbose} - ${tests.seed} - ${tests.failfast} - - - ${tests.iters} - ${tests.maxfailures} - ${tests.failfast} - ${tests.class} - ${tests.method} - ${tests.nightly} - ${tests.badapples} - ${tests.weekly} - ${tests.slow} - ${tests.awaitsfix} - ${tests.slow} - ${tests.timeoutSuite} - ${tests.showSuccess} - ${tests.integration} - ${tests.cluster_seed} - ${tests.client.ratio} - ${env.ES_TEST_LOCAL} - ${es.node.mode} - ${es.logger.level} - true - - - - org.apache.maven.plugins maven-surefire-plugin - 2.15 - - true - org.apache.maven.plugins maven-source-plugin - 2.1.2 - - - attach-sources - - jar - - - + org.apache.maven.plugins maven-assembly-plugin - 2.2.2 false ${project.build.directory}/releases/ From fae56e39f9eaa92c19b298592716b2f1f66475b7 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 2 Jan 2015 18:42:56 +0100 Subject: [PATCH 55/73] Add sonatype snapshot repository --- pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pom.xml b/pom.xml index 7ccad2ed947..df9c1a723b3 100644 --- a/pom.xml +++ b/pom.xml @@ -120,4 +120,12 @@ + + + + oss-snapshots + Sonatype OSS Snapshots + https://oss.sonatype.org/content/repositories/snapshots/ + + From 4cfb957f3778cf166d82c1cc57b5f5209333e0df Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 12 Feb 2015 09:53:28 +0100 Subject: [PATCH 56/73] update documentation with release 2.3.1 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e57a99d3317..0436db22fe9 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The Python (jython) language plugin allows to have `python` as the language of s In order to install the plugin, simply run: ```sh -bin/plugin -install elasticsearch/elasticsearch-lang-python/2.4.1 +bin/plugin install elasticsearch/elasticsearch-lang-python/2.3.1 ``` You need to install a version matching your Elasticsearch version: @@ -16,7 +16,7 @@ You need to install a version matching your Elasticsearch version: | master | Build from source | See below | | es-1.x | Build from source | [2.5.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.x/#version-250-snapshot-for-elasticsearch-1x) | | es-1.4 | 2.4.1 | [2.4.1](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.4.1/#version-241-for-elasticsearch-14) | -| es-1.3 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.0/#python-lang-plugin-for-elasticsearch) | +| es-1.3 | 2.3.1 | [2.3.1](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.1/#version-231-for-elasticsearch-13) | | es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.2.0/#python-lang-plugin-for-elasticsearch) | | es-1.0 | 2.0.0 | [2.0.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.0.0/#python-lang-plugin-for-elasticsearch) | | es-0.90 | 1.0.0 | [1.0.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v1.0.0/#python-lang-plugin-for-elasticsearch) | From 42573fe6fcb66c5fdb11115a11ca6d9f4a7dedac Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 12 Feb 2015 10:01:13 +0100 Subject: [PATCH 57/73] Fix version number for elasticsearch < 1.3.5 Related to #26 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0436db22fe9..1aaf18b1c34 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ You need to install a version matching your Elasticsearch version: | es-1.x | Build from source | [2.5.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.x/#version-250-snapshot-for-elasticsearch-1x) | | es-1.4 | 2.4.1 | [2.4.1](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.4.1/#version-241-for-elasticsearch-14) | | es-1.3 | 2.3.1 | [2.3.1](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.1/#version-231-for-elasticsearch-13) | +| < 1.3.5 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.0/#version-230-for-elasticsearch-13) | | es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.2.0/#python-lang-plugin-for-elasticsearch) | | es-1.0 | 2.0.0 | [2.0.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.0.0/#python-lang-plugin-for-elasticsearch) | | es-0.90 | 1.0.0 | [1.0.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v1.0.0/#python-lang-plugin-for-elasticsearch) | From bc8073a40d51bf317ac7e69f94bd2b49f6d774a6 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 16 Mar 2015 16:47:41 -0700 Subject: [PATCH 58/73] create `es-1.5` branch --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1aaf18b1c34..f243987ceaa 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ You need to install a version matching your Elasticsearch version: | elasticsearch | Python Lang Plugin | Docs | |---------------|-----------------------|------------| | master | Build from source | See below | -| es-1.x | Build from source | [2.5.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.x/#version-250-snapshot-for-elasticsearch-1x) | +| es-1.x | Build from source | [2.6.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.x/#version-260-snapshot-for-elasticsearch-1x) | +| es-1.5 | Build from source | [2.5.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.5/#version-250-snapshot-for-elasticsearch-15) | | es-1.4 | 2.4.1 | [2.4.1](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.4.1/#version-241-for-elasticsearch-14) | | es-1.3 | 2.3.1 | [2.3.1](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.1/#version-231-for-elasticsearch-13) | | < 1.3.5 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.0/#version-230-for-elasticsearch-13) | From 4e371228567e91c7b89b1bbaad122572b8762124 Mon Sep 17 00:00:00 2001 From: Njal Karevoll Date: Thu, 26 Mar 2015 19:09:04 +0100 Subject: [PATCH 59/73] Update plugin description Updates wording to (hopefully) better reflect what it does and removes reference to Javascript. --- src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java b/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java index 1e82ea61851..78f05311a6f 100644 --- a/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java +++ b/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java @@ -35,7 +35,7 @@ public class PythonPlugin extends AbstractPlugin { @Override public String description() { - return "Python plugin allowing to add javascript scripting support"; + return "Adds support for writing scripts in Python"; } public void onModule(ScriptModule module) { From 655d8e4287621aabd6bfb4ceb551b1aa67570741 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Sat, 28 Mar 2015 14:09:16 +0100 Subject: [PATCH 60/73] Move parent after artifact coordinates --- pom.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index df9c1a723b3..df3285398c0 100644 --- a/pom.xml +++ b/pom.xml @@ -4,12 +4,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - - org.elasticsearch - elasticsearch-parent - 2.0.0-SNAPSHOT - - org.elasticsearch elasticsearch-lang-python 3.0.0-SNAPSHOT @@ -31,6 +25,12 @@ http://github.com/elasticsearch/elasticsearch-lang-python + + org.elasticsearch + elasticsearch-parent + 2.0.0-SNAPSHOT + + From f7c0fbaee3540cb1749abb1cc934a7befd94704d Mon Sep 17 00:00:00 2001 From: David Pilato Date: Sat, 28 Mar 2015 14:06:41 +0100 Subject: [PATCH 61/73] Update owner to elastic (cherry picked from commit 283b92f) (cherry picked from commit 49445b3) --- pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index df3285398c0..ed85f9227dd 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ jar Elasticsearch Python language plugin The Python language plugin allows to have python as the language of scripts to execute. - https://github.com/elasticsearch/elasticsearch-lang-python/ + https://github.com/elastic/elasticsearch-lang-python/ 2009 @@ -20,9 +20,9 @@ - scm:git:git@github.com:elasticsearch/elasticsearch-lang-python.git - scm:git:git@github.com:elasticsearch/elasticsearch-lang-python.git - http://github.com/elasticsearch/elasticsearch-lang-python + scm:git:git@github.com:elastic/elasticsearch-lang-python.git + scm:git:git@github.com:elastic/elasticsearch-lang-python.git + http://github.com/elastic/elasticsearch-lang-python From 975feb1cbb68a77f7728dd77d5b15c43b428a1ab Mon Sep 17 00:00:00 2001 From: David Pilato Date: Sat, 28 Mar 2015 14:34:03 +0100 Subject: [PATCH 62/73] update documentation with release 2.5.0 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f243987ceaa..6e6116b1028 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The Python (jython) language plugin allows to have `python` as the language of s In order to install the plugin, simply run: ```sh -bin/plugin install elasticsearch/elasticsearch-lang-python/2.3.1 +bin/plugin install elasticsearch/elasticsearch-lang-python/2.5.0 ``` You need to install a version matching your Elasticsearch version: @@ -15,7 +15,7 @@ You need to install a version matching your Elasticsearch version: |---------------|-----------------------|------------| | master | Build from source | See below | | es-1.x | Build from source | [2.6.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.x/#version-260-snapshot-for-elasticsearch-1x) | -| es-1.5 | Build from source | [2.5.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.5/#version-250-snapshot-for-elasticsearch-15) | +| es-1.5 | 2.5.0 | [2.5.0](https://github.com/elastic/elasticsearch-lang-python/tree/v2.5.0/#version-250-for-elasticsearch-15) | | es-1.4 | 2.4.1 | [2.4.1](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.4.1/#version-241-for-elasticsearch-14) | | es-1.3 | 2.3.1 | [2.3.1](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.1/#version-231-for-elasticsearch-13) | | < 1.3.5 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.0/#version-230-for-elasticsearch-13) | From 410e0fa6990bba63a16d396156442fc36dcbdae8 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 9 Apr 2015 14:39:04 +0200 Subject: [PATCH 63/73] Fix compilation. See https://github.com/elastic/elasticsearch/pull/10389 for background. --- .../python/PythonScriptEngineService.java | 42 +++++++++++-------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java index 54e1543011b..6138453925e 100644 --- a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java @@ -19,13 +19,22 @@ package org.elasticsearch.script.python; +import java.io.IOException; +import java.util.Map; + import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Scorer; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.script.*; +import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.LeafSearchScript; +import org.elasticsearch.script.ScoreAccessor; +import org.elasticsearch.script.ScriptEngineService; +import org.elasticsearch.script.SearchScript; +import org.elasticsearch.search.lookup.LeafSearchLookup; import org.elasticsearch.search.lookup.SearchLookup; import org.python.core.Py; import org.python.core.PyCode; @@ -33,8 +42,6 @@ import org.python.core.PyObject; import org.python.core.PyStringMap; import org.python.util.PythonInterpreter; -import java.util.Map; - /** * */ @@ -76,8 +83,14 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri } @Override - public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable Map vars) { - return new PythonSearchScript((PyCode) compiledScript, vars, lookup); + public SearchScript search(final Object compiledScript, final SearchLookup lookup, @Nullable final Map vars) { + return new SearchScript() { + @Override + public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException { + final LeafSearchLookup leafLookup = lookup.getLeafSearchLookup(context); + return new PythonSearchScript((PyCode) compiledScript, vars, leafLookup); + } + }; } @Override @@ -143,15 +156,15 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri } } - public class PythonSearchScript implements SearchScript { + public class PythonSearchScript implements LeafSearchScript { private final PyCode code; private final PyStringMap pyVars; - private final SearchLookup lookup; + private final LeafSearchLookup lookup; - public PythonSearchScript(PyCode code, Map vars, SearchLookup lookup) { + public PythonSearchScript(PyCode code, Map vars, LeafSearchLookup lookup) { this.code = code; this.pyVars = new PyStringMap(); for (Map.Entry entry : lookup.asMap().entrySet()) { @@ -171,18 +184,13 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri } @Override - public void setNextReader(LeafReaderContext context) { - lookup.setNextReader(context); + public void setDocument(int doc) { + lookup.setDocument(doc); } @Override - public void setNextDocId(int doc) { - lookup.setNextDocId(doc); - } - - @Override - public void setNextSource(Map source) { - lookup.source().setNextSource(source); + public void setSource(Map source) { + lookup.source().setSource(source); } @Override From 8eeec50ace76ca39b51a4cce4df8c63daa960153 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 11 May 2015 20:56:19 -0400 Subject: [PATCH 64/73] enable security manager in tests and upgrade to final jython 2.7.0 The previous rc version would try to execute /usr/bin/id... --- dev-tools/tests.policy | 54 ------------------- pom.xml | 3 +- .../python/PythonScriptSearchTests.java | 7 ++- 3 files changed, 5 insertions(+), 59 deletions(-) delete mode 100644 dev-tools/tests.policy diff --git a/dev-tools/tests.policy b/dev-tools/tests.policy deleted file mode 100644 index 6afb5025840..00000000000 --- a/dev-tools/tests.policy +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -// Policy file to prevent tests from writing outside the test sandbox directory -// PLEASE NOTE: You may need to enable other permissions when new tests are added, -// everything not allowed here is forbidden! - -grant { - // permissions for file access, write access only to sandbox: - permission java.io.FilePermission "<>", "read,execute"; - permission java.io.FilePermission "${junit4.childvm.cwd}", "read,execute,write"; - permission java.io.FilePermission "${junit4.childvm.cwd}${/}-", "read,execute,write,delete"; - permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,execute,write,delete"; - permission groovy.security.GroovyCodeSourcePermission "/groovy/script"; - - // Allow connecting to the internet anywhere - permission java.net.SocketPermission "*", "accept,listen,connect,resolve"; - - // Basic permissions needed for Lucene / Elasticsearch to work: - permission java.util.PropertyPermission "*", "read,write"; - permission java.lang.reflect.ReflectPermission "*"; - permission java.lang.RuntimePermission "*"; - - // These two *have* to be spelled out a separate - permission java.lang.management.ManagementPermission "control"; - permission java.lang.management.ManagementPermission "monitor"; - - permission java.net.NetPermission "*"; - permission java.util.logging.LoggingPermission "control"; - permission javax.management.MBeanPermission "*", "*"; - permission javax.management.MBeanServerPermission "*"; - permission javax.management.MBeanTrustPermission "*"; - - // Needed for some things in DNS caching in the JVM - permission java.security.SecurityPermission "getProperty.networkaddress.cache.ttl"; - permission java.security.SecurityPermission "getProperty.networkaddress.cache.negative.ttl"; - -}; diff --git a/pom.xml b/pom.xml index ed85f9227dd..f113a77d8f9 100644 --- a/pom.xml +++ b/pom.xml @@ -33,6 +33,7 @@ + true @@ -57,7 +58,7 @@ org.python jython-standalone - 2.7-b3 + 2.7.0 diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java index 3543bea7654..62446eac392 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java @@ -41,7 +41,6 @@ import java.util.Map; import static org.elasticsearch.client.Requests.searchRequest; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.FilterBuilders.scriptFilter; import static org.elasticsearch.index.query.QueryBuilders.*; import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; @@ -83,7 +82,7 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { logger.info(" --> running doc['num1'].value > 1"); SearchResponse response = client().prepareSearch() - .setQuery(filteredQuery(matchAllQuery(), scriptFilter("doc['num1'].value > 1").lang("python"))) + .setQuery(filteredQuery(matchAllQuery(), scriptQuery("doc['num1'].value > 1").lang("python"))) .addSort("num1", SortOrder.ASC) .addScriptField("sNum1", "python", "doc['num1'].value", null) .execute().actionGet(); @@ -96,7 +95,7 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { logger.info(" --> running doc['num1'].value > param1"); response = client().prepareSearch() - .setQuery(filteredQuery(matchAllQuery(), scriptFilter("doc['num1'].value > param1").lang("python").addParam("param1", 2))) + .setQuery(filteredQuery(matchAllQuery(), scriptQuery("doc['num1'].value > param1").lang("python").addParam("param1", 2))) .addSort("num1", SortOrder.ASC) .addScriptField("sNum1", "python", "doc['num1'].value", null) .execute().actionGet(); @@ -107,7 +106,7 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { logger.info(" --> running doc['num1'].value > param1"); response = client().prepareSearch() - .setQuery(filteredQuery(matchAllQuery(), scriptFilter("doc['num1'].value > param1").lang("python").addParam("param1", -1))) + .setQuery(filteredQuery(matchAllQuery(), scriptQuery("doc['num1'].value > param1").lang("python").addParam("param1", -1))) .addSort("num1", SortOrder.ASC) .addScriptField("sNum1", "python", "doc['num1'].value", null) .execute().actionGet(); From 1999fba0e8f2dac244bc77caa034b04f0efa1071 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 13 May 2015 23:36:19 -0400 Subject: [PATCH 65/73] remove unnecessary property --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index f113a77d8f9..987cc0b6b10 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,6 @@ - true From 2579af4fb57126690bfa0ed9c6c529c1274d4df3 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 21 May 2015 10:37:06 -0400 Subject: [PATCH 66/73] engage forbidden apis --- pom.xml | 47 ++--------------------------------------------- 1 file changed, 2 insertions(+), 45 deletions(-) diff --git a/pom.xml b/pom.xml index 987cc0b6b10..d3881223617 100644 --- a/pom.xml +++ b/pom.xml @@ -40,14 +40,12 @@ org.hamcrest hamcrest-all - - com.carrotsearch.randomizedtesting - randomizedtesting-runner - + org.apache.lucene lucene-test-framework + org.elasticsearch elasticsearch @@ -72,51 +70,10 @@ - - - src/main/resources - true - - **/*.properties - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - com.carrotsearch.randomizedtesting - junit4-maven-plugin - - - - org.apache.maven.plugins - maven-surefire-plugin - - - org.apache.maven.plugins - maven-source-plugin - org.apache.maven.plugins maven-assembly-plugin - - false - ${project.build.directory}/releases/ - - ${basedir}/src/main/assemblies/plugin.xml - - - - - package - - single - - - From 79a8428aa2f53d8afebfee60df510dbcd2c1cfa8 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 21 May 2015 16:52:05 -0400 Subject: [PATCH 67/73] remove duplicate test config --- pom.xml | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/pom.xml b/pom.xml index d3881223617..fbab7a0209e 100644 --- a/pom.xml +++ b/pom.xml @@ -36,37 +36,12 @@ - - org.hamcrest - hamcrest-all - - - - org.apache.lucene - lucene-test-framework - - - - org.elasticsearch - elasticsearch - - org.python jython-standalone 2.7.0 - - - log4j - log4j - - - org.elasticsearch - elasticsearch - test-jar - From 65253a4bc86d3e4fc7e4e93c4ff18796d643f43d Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 21 May 2015 22:55:42 -0400 Subject: [PATCH 68/73] switch to plugin pom --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fbab7a0209e..0a7c20de918 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ org.elasticsearch - elasticsearch-parent + elasticsearch-plugin 2.0.0-SNAPSHOT From fd035c556c698f37f185fa9c9eed61321c352eb0 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Fri, 22 May 2015 09:32:19 -0400 Subject: [PATCH 69/73] remove logging properties --- src/test/resources/log4j.properties | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 src/test/resources/log4j.properties diff --git a/src/test/resources/log4j.properties b/src/test/resources/log4j.properties deleted file mode 100644 index 497c97f9959..00000000000 --- a/src/test/resources/log4j.properties +++ /dev/null @@ -1,5 +0,0 @@ -log4j.rootLogger=INFO, out - -log4j.appender.out=org.apache.log4j.ConsoleAppender -log4j.appender.out.layout=org.apache.log4j.PatternLayout -log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n From a86d9e790846c4add348a918bac12ef420fc16f8 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Tue, 26 May 2015 09:34:33 -0400 Subject: [PATCH 70/73] Absorb ImmutableSettings into Settings --- .../script/python/PythonScriptEngineTests.java | 4 ++-- .../script/python/PythonScriptMultiThreadedTest.java | 8 ++++---- .../script/python/PythonScriptSearchTests.java | 3 +-- .../java/org/elasticsearch/script/python/SimpleBench.java | 4 ++-- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java index 92435ea9be3..915c502636f 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java @@ -21,7 +21,7 @@ package org.elasticsearch.script.python; import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.collect.MapBuilder; -import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.test.ElasticsearchTestCase; import org.junit.After; @@ -43,7 +43,7 @@ public class PythonScriptEngineTests extends ElasticsearchTestCase { @Before public void setup() { - se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS); + se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); } @After diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java b/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java index 0cd49cd21fe..903f6318761 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java @@ -19,7 +19,7 @@ package org.elasticsearch.script.python; -import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.test.ElasticsearchTestCase; import org.junit.After; @@ -48,7 +48,7 @@ public class PythonScriptMultiThreadedTest extends ElasticsearchTestCase { @Test public void testExecutableNoRuntimeParams() throws Exception { - final PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS); + final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); final Object compiled = se.compile("x + y"); final AtomicBoolean failed = new AtomicBoolean(); @@ -91,7 +91,7 @@ public class PythonScriptMultiThreadedTest extends ElasticsearchTestCase { // @Test public void testExecutableWithRuntimeParams() throws Exception { -// final PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS); +// final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); // final Object compiled = se.compile("x + y"); // final AtomicBoolean failed = new AtomicBoolean(); // @@ -134,7 +134,7 @@ public class PythonScriptMultiThreadedTest extends ElasticsearchTestCase { @Test public void testExecute() throws Exception { - final PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS); + final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); final Object compiled = se.compile("x + y"); final AtomicBoolean failed = new AtomicBoolean(); diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java index 62446eac392..0ff68beea3e 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java @@ -21,7 +21,6 @@ package org.elasticsearch.script.python; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.plugins.PluginsService; @@ -57,7 +56,7 @@ public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { @Override protected Settings nodeSettings(int nodeOrdinal) { - return ImmutableSettings.builder() + return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) .build(); diff --git a/src/test/java/org/elasticsearch/script/python/SimpleBench.java b/src/test/java/org/elasticsearch/script/python/SimpleBench.java index 7877187c866..583bab163fa 100644 --- a/src/test/java/org/elasticsearch/script/python/SimpleBench.java +++ b/src/test/java/org/elasticsearch/script/python/SimpleBench.java @@ -20,7 +20,7 @@ package org.elasticsearch.script.python; import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.ExecutableScript; import java.util.HashMap; @@ -32,7 +32,7 @@ import java.util.Map; public class SimpleBench { public static void main(String[] args) { - PythonScriptEngineService se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS); + PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); Object compiled = se.compile("x + y"); Map vars = new HashMap(); From 23726ece0039013e670aa154d36ce34eb29593eb Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 27 May 2015 19:53:49 -0400 Subject: [PATCH 71/73] tone down test --- .../script/python/PythonScriptMultiThreadedTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java b/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java index 903f6318761..9d53507388b 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java @@ -52,7 +52,7 @@ public class PythonScriptMultiThreadedTest extends ElasticsearchTestCase { final Object compiled = se.compile("x + y"); final AtomicBoolean failed = new AtomicBoolean(); - Thread[] threads = new Thread[50]; + Thread[] threads = new Thread[4]; final CountDownLatch latch = new CountDownLatch(threads.length); final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1); for (int i = 0; i < threads.length; i++) { @@ -68,7 +68,7 @@ public class PythonScriptMultiThreadedTest extends ElasticsearchTestCase { vars.put("x", x); vars.put("y", y); ExecutableScript script = se.executable(compiled, vars); - for (int i = 0; i < 100000; i++) { + for (int i = 0; i < 10000; i++) { long result = ((Number) script.run()).longValue(); assertThat(result, equalTo(addition)); } @@ -138,7 +138,7 @@ public class PythonScriptMultiThreadedTest extends ElasticsearchTestCase { final Object compiled = se.compile("x + y"); final AtomicBoolean failed = new AtomicBoolean(); - Thread[] threads = new Thread[50]; + Thread[] threads = new Thread[4]; final CountDownLatch latch = new CountDownLatch(threads.length); final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1); for (int i = 0; i < threads.length; i++) { @@ -148,7 +148,7 @@ public class PythonScriptMultiThreadedTest extends ElasticsearchTestCase { try { barrier.await(); Map runtimeVars = new HashMap(); - for (int i = 0; i < 100000; i++) { + for (int i = 0; i < 10000; i++) { long x = ThreadLocalRandom.current().nextInt(); long y = ThreadLocalRandom.current().nextInt(); long addition = x + y; From f77bcf926c06d678678f49c53cac6a8f4761d4fb Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 8 Jun 2015 12:01:43 +0200 Subject: [PATCH 72/73] [TEST] Don't use shaded API --- .../script/python/PythonScriptEngineTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java index 915c502636f..1621d22ac01 100644 --- a/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java +++ b/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.script.python; -import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.ExecutableScript; @@ -28,6 +27,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -66,7 +66,7 @@ public class PythonScriptEngineTests extends ElasticsearchTestCase { Map vars = new HashMap(); Map obj2 = MapBuilder.newMapBuilder().put("prop2", "value2").map(); - Map obj1 = MapBuilder.newMapBuilder().put("prop1", "value1").put("obj2", obj2).put("l", Lists.newArrayList("2", "1")).map(); + Map obj1 = MapBuilder.newMapBuilder().put("prop1", "value1").put("obj2", obj2).put("l", Arrays.asList("2", "1")).map(); vars.put("obj1", obj1); Object o = se.execute(se.compile("obj1"), vars); assertThat(o, instanceOf(Map.class)); @@ -101,7 +101,7 @@ public class PythonScriptEngineTests extends ElasticsearchTestCase { Map vars = new HashMap(); Map obj2 = MapBuilder.newMapBuilder().put("prop2", "value2").map(); Map obj1 = MapBuilder.newMapBuilder().put("prop1", "value1").put("obj2", obj2).map(); - vars.put("l", Lists.newArrayList("1", "2", "3", obj1)); + vars.put("l", Arrays.asList("1", "2", "3", obj1)); // Object o = se.execute(se.compile("l.length"), vars); // assertThat(((Number) o).intValue(), equalTo(4)); From a1b410652da7ff8a6b3568342cf3d78cb1519135 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Tue, 9 Jun 2015 15:17:22 +0200 Subject: [PATCH 73/73] add lang-python module --- .gitignore | 13 -- CONTRIBUTING.md | 98 --------- LICENSE.txt | 202 ------------------ dev-tools/release.py | 134 ------------ README.md => plugins/lang-python/README.md | 0 pom.xml => plugins/lang-python/pom.xml | 25 +-- .../src}/main/assemblies/plugin.xml | 0 .../plugin/python/PythonPlugin.java | 0 .../python/PythonScriptEngineService.java | 0 .../src}/main/resources/es-plugin.properties | 0 .../python/PythonScriptEngineTests.java | 0 .../python/PythonScriptMultiThreadedTest.java | 0 .../python/PythonScriptSearchTests.java | 0 .../script/python/SimpleBench.java | 0 14 files changed, 2 insertions(+), 470 deletions(-) delete mode 100644 .gitignore delete mode 100644 CONTRIBUTING.md delete mode 100644 LICENSE.txt delete mode 100644 dev-tools/release.py rename README.md => plugins/lang-python/README.md (100%) rename pom.xml => plugins/lang-python/pom.xml (57%) rename {src => plugins/lang-python/src}/main/assemblies/plugin.xml (100%) rename {src => plugins/lang-python/src}/main/java/org/elasticsearch/plugin/python/PythonPlugin.java (100%) rename {src => plugins/lang-python/src}/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java (100%) rename {src => plugins/lang-python/src}/main/resources/es-plugin.properties (100%) rename {src => plugins/lang-python/src}/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java (100%) rename {src => plugins/lang-python/src}/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java (100%) rename {src => plugins/lang-python/src}/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java (100%) rename {src => plugins/lang-python/src}/test/java/org/elasticsearch/script/python/SimpleBench.java (100%) diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 9533848e238..00000000000 --- a/.gitignore +++ /dev/null @@ -1,13 +0,0 @@ -/data -/work -/logs -/.idea -/target -.DS_Store -*.iml -/.project -/.settings -/.classpath -/plugin_tools -/.local-execution-hints.log -/.local-*-execution-hints.log diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 0c59bbbc6ea..00000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,98 +0,0 @@ -Contributing to elasticsearch -============================= - -Elasticsearch is an open source project and we love to receive contributions from our community — you! There are many ways to contribute, from writing tutorials or blog posts, improving the documentation, submitting bug reports and feature requests or writing code which can be incorporated into Elasticsearch itself. - -Bug reports ------------ - -If you think you have found a bug in Elasticsearch, first make sure that you are testing against the [latest version of Elasticsearch](http://www.elasticsearch.org/download/) - your issue may already have been fixed. If not, search our [issues list](https://github.com/elasticsearch/elasticsearch/issues) on GitHub in case a similar issue has already been opened. - -It is very helpful if you can prepare a reproduction of the bug. In other words, provide a small test case which we can run to confirm your bug. It makes it easier to find the problem and to fix it. Test cases should be provided as `curl` commands which we can copy and paste into a terminal to run it locally, for example: - -```sh -# delete the index -curl -XDELETE localhost:9200/test - -# insert a document -curl -XPUT localhost:9200/test/test/1 -d '{ - "title": "test document" -}' - -# this should return XXXX but instead returns YYY -curl .... -``` - -Provide as much information as you can. You may think that the problem lies with your query, when actually it depends on how your data is indexed. The easier it is for us to recreate your problem, the faster it is likely to be fixed. - -Feature requests ----------------- - -If you find yourself wishing for a feature that doesn't exist in Elasticsearch, you are probably not alone. There are bound to be others out there with similar needs. Many of the features that Elasticsearch has today have been added because our users saw the need. -Open an issue on our [issues list](https://github.com/elasticsearch/elasticsearch/issues) on GitHub which describes the feature you would like to see, why you need it, and how it should work. - -Contributing code and documentation changes -------------------------------------------- - -If you have a bugfix or new feature that you would like to contribute to Elasticsearch, please find or open an issue about it first. Talk about what you would like to do. It may be that somebody is already working on it, or that there are particular issues that you should know about before implementing the change. - -We enjoy working with contributors to get their code accepted. There are many approaches to fixing a problem and it is important to find the best approach before writing too much code. - -The process for contributing to any of the [Elasticsearch repositories](https://github.com/elasticsearch/) is similar. Details for individual projects can be found below. - -### Fork and clone the repository - -You will need to fork the main Elasticsearch code or documentation repository and clone it to your local machine. See -[github help page](https://help.github.com/articles/fork-a-repo) for help. - -Further instructions for specific projects are given below. - -### Submitting your changes - -Once your changes and tests are ready to submit for review: - -1. Test your changes -Run the test suite to make sure that nothing is broken. - -2. Sign the Contributor License Agreement -Please make sure you have signed our [Contributor License Agreement](http://www.elasticsearch.org/contributor-agreement/). We are not asking you to assign copyright to us, but to give us the right to distribute your code without restriction. We ask this of all contributors in order to assure our users of the origin and continuing existence of the code. You only need to sign the CLA once. - -3. Rebase your changes -Update your local repository with the most recent code from the main Elasticsearch repository, and rebase your branch on top of the latest master branch. We prefer your changes to be squashed into a single commit. - -4. Submit a pull request -Push your local changes to your forked copy of the repository and [submit a pull request](https://help.github.com/articles/using-pull-requests). In the pull request, describe what your changes do and mention the number of the issue where discussion has taken place, eg "Closes #123". - -Then sit back and wait. There will probably be discussion about the pull request and, if any changes are needed, we would love to work with you to get your pull request merged into Elasticsearch. - - -Contributing to the Elasticsearch plugin ----------------------------------------- - -**Repository:** [https://github.com/elasticsearch/elasticsearch-lang-python](https://github.com/elasticsearch/elasticsearch-lang-python) - -Make sure you have [Maven](http://maven.apache.org) installed, as Elasticsearch uses it as its build system. Integration with IntelliJ and Eclipse should work out of the box. Eclipse users can automatically configure their IDE by running `mvn eclipse:eclipse` and then importing the project into their workspace: `File > Import > Existing project into workspace`. - -Please follow these formatting guidelines: - -* Java indent is 4 spaces -* Line width is 140 characters -* The rest is left to Java coding standards -* Disable “auto-format on save” to prevent unnecessary format changes. This makes reviews much harder as it generates unnecessary formatting changes. If your IDE supports formatting only modified chunks that is fine to do. - -To create a distribution from the source, simply run: - -```sh -cd elasticsearch-lang-python/ -mvn clean package -DskipTests -``` - -You will find the newly built packages under: `./target/releases/`. - -Before submitting your changes, run the test suite to make sure that nothing is broken, with: - -```sh -mvn clean test -``` - -Source: [Contributing to elasticsearch](http://www.elasticsearch.org/contributing-to-elasticsearch/) diff --git a/LICENSE.txt b/LICENSE.txt deleted file mode 100644 index d6456956733..00000000000 --- a/LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/dev-tools/release.py b/dev-tools/release.py deleted file mode 100644 index edcc637d068..00000000000 --- a/dev-tools/release.py +++ /dev/null @@ -1,134 +0,0 @@ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on -# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - -import datetime -import os -import shutil -import sys -import time -import urllib -import urllib.request -import zipfile - -from os.path import dirname, abspath - -""" - This tool builds a release from the a given elasticsearch plugin branch. - - It is basically a wrapper on top of launch_release.py which: - - - tries to get a more recent version of launch_release.py in ... - - download it if needed - - launch it passing all arguments to it, like: - - $ python3 dev_tools/release.py --branch master --publish --remote origin - - Important options: - - # Dry run - $ python3 dev_tools/release.py - - # Dry run without tests - python3 dev_tools/release.py --skiptests - - # Release, publish artifacts and announce - $ python3 dev_tools/release.py --publish - - See full documentation in launch_release.py -""" -env = os.environ - -# Change this if the source repository for your scripts is at a different location -SOURCE_REPO = 'elasticsearch/elasticsearch-plugins-script' -# We define that we should download again the script after 1 days -SCRIPT_OBSOLETE_DAYS = 1 -# We ignore in master.zip file the following files -IGNORED_FILES = ['.gitignore', 'README.md'] - - -ROOT_DIR = abspath(os.path.join(abspath(dirname(__file__)), '../')) -TARGET_TOOLS_DIR = ROOT_DIR + '/plugin_tools' -DEV_TOOLS_DIR = ROOT_DIR + '/dev-tools' -BUILD_RELEASE_FILENAME = 'release.zip' -BUILD_RELEASE_FILE = TARGET_TOOLS_DIR + '/' + BUILD_RELEASE_FILENAME -SOURCE_URL = 'https://github.com/%s/archive/master.zip' % SOURCE_REPO - -# Download a recent version of the release plugin tool -try: - os.mkdir(TARGET_TOOLS_DIR) - print('directory %s created' % TARGET_TOOLS_DIR) -except FileExistsError: - pass - - -try: - # we check latest update. If we ran an update recently, we - # are not going to check it again - download = True - - try: - last_download_time = datetime.datetime.fromtimestamp(os.path.getmtime(BUILD_RELEASE_FILE)) - if (datetime.datetime.now()-last_download_time).days < SCRIPT_OBSOLETE_DAYS: - download = False - except FileNotFoundError: - pass - - if download: - urllib.request.urlretrieve(SOURCE_URL, BUILD_RELEASE_FILE) - with zipfile.ZipFile(BUILD_RELEASE_FILE) as myzip: - for member in myzip.infolist(): - filename = os.path.basename(member.filename) - # skip directories - if not filename: - continue - if filename in IGNORED_FILES: - continue - - # copy file (taken from zipfile's extract) - source = myzip.open(member.filename) - target = open(os.path.join(TARGET_TOOLS_DIR, filename), "wb") - with source, target: - shutil.copyfileobj(source, target) - # We keep the original date - date_time = time.mktime(member.date_time + (0, 0, -1)) - os.utime(os.path.join(TARGET_TOOLS_DIR, filename), (date_time, date_time)) - print('plugin-tools updated from %s' % SOURCE_URL) -except urllib.error.HTTPError: - pass - - -# Let see if we need to update the release.py script itself -source_time = os.path.getmtime(TARGET_TOOLS_DIR + '/release.py') -repo_time = os.path.getmtime(DEV_TOOLS_DIR + '/release.py') -if source_time > repo_time: - input('release.py needs an update. Press a key to update it...') - shutil.copyfile(TARGET_TOOLS_DIR + '/release.py', DEV_TOOLS_DIR + '/release.py') - -# We can launch the build process -try: - PYTHON = 'python' - # make sure python3 is used if python3 is available - # some systems use python 2 as default - os.system('python3 --version > /dev/null 2>&1') - PYTHON = 'python3' -except RuntimeError: - pass - -release_args = '' -for x in range(1, len(sys.argv)): - release_args += ' ' + sys.argv[x] - -os.system('%s %s/build_release.py %s' % (PYTHON, TARGET_TOOLS_DIR, release_args)) diff --git a/README.md b/plugins/lang-python/README.md similarity index 100% rename from README.md rename to plugins/lang-python/README.md diff --git a/pom.xml b/plugins/lang-python/pom.xml similarity index 57% rename from pom.xml rename to plugins/lang-python/pom.xml index 0a7c20de918..1d52498628c 100644 --- a/pom.xml +++ b/plugins/lang-python/pom.xml @@ -4,26 +4,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - org.elasticsearch + org.elasticsearch.plugin elasticsearch-lang-python - 3.0.0-SNAPSHOT + jar Elasticsearch Python language plugin The Python language plugin allows to have python as the language of scripts to execute. - https://github.com/elastic/elasticsearch-lang-python/ - 2009 - - - The Apache Software License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - scm:git:git@github.com:elastic/elasticsearch-lang-python.git - scm:git:git@github.com:elastic/elasticsearch-lang-python.git - http://github.com/elastic/elasticsearch-lang-python - org.elasticsearch @@ -53,11 +39,4 @@ - - - oss-snapshots - Sonatype OSS Snapshots - https://oss.sonatype.org/content/repositories/snapshots/ - - diff --git a/src/main/assemblies/plugin.xml b/plugins/lang-python/src/main/assemblies/plugin.xml similarity index 100% rename from src/main/assemblies/plugin.xml rename to plugins/lang-python/src/main/assemblies/plugin.xml diff --git a/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java b/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java similarity index 100% rename from src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java rename to plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java diff --git a/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java similarity index 100% rename from src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java rename to plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java diff --git a/src/main/resources/es-plugin.properties b/plugins/lang-python/src/main/resources/es-plugin.properties similarity index 100% rename from src/main/resources/es-plugin.properties rename to plugins/lang-python/src/main/resources/es-plugin.properties diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java similarity index 100% rename from src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java rename to plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java similarity index 100% rename from src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java rename to plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java similarity index 100% rename from src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java rename to plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java diff --git a/src/test/java/org/elasticsearch/script/python/SimpleBench.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/SimpleBench.java similarity index 100% rename from src/test/java/org/elasticsearch/script/python/SimpleBench.java rename to plugins/lang-python/src/test/java/org/elasticsearch/script/python/SimpleBench.java