Scripts: Allow to register native scripts (Java) for better script execution performance, closes #752.

modules/benchmark/micro/src/main/java/org/elasticsearch/benchmark/percolator/EmbeddedPercolatorBenchmarkTest.java

@@ -62,7 +62,7 @@ public class EmbeddedPercolatorBenchmarkTest {
         Injector injector = new ModulesBuilder().add(
                 new SettingsModule(settings),
                 new ThreadPoolModule(settings),
-                new ScriptModule(),
+                new ScriptModule(settings),
                 new MapperServiceModule(),
                 new IndexSettingsModule(settings),
                 new IndexCacheModule(settings),

modules/elasticsearch/src/main/java/org/elasticsearch/node/internal/InternalNode.java

@@ -119,7 +119,7 @@ public final class InternalNode implements Node {
         modules.add(new NodeModule(this));
         modules.add(new NetworkModule());
         modules.add(new NodeCacheModule(settings));
-        modules.add(new ScriptModule());
+        modules.add(new ScriptModule(settings));
         modules.add(new JmxModule(settings));
         modules.add(new EnvironmentModule(environment));
         modules.add(new NodeEnvironmentModule());

modules/elasticsearch/src/main/java/org/elasticsearch/script/AbstractDoubleSearchScript.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+public abstract class AbstractDoubleSearchScript extends AbstractSearchScript {
+
+    @Override public Object run() {
+        return runAsDouble();
+    }
+
+    @Override public abstract double runAsDouble();
+
+    @Override public long runAsLong() {
+        return (long) runAsDouble();
+    }
+
+    @Override public float runAsFloat() {
+        return (float) runAsDouble();
+    }
+}

modules/elasticsearch/src/main/java/org/elasticsearch/script/AbstractExecutableScript.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+public abstract class AbstractExecutableScript implements ExecutableScript {
+
+    @Override public void setNextVar(String name, Object value) {
+    }
+
+    @Override public Object unwrap(Object value) {
+        return value;
+    }
+}

modules/elasticsearch/src/main/java/org/elasticsearch/script/AbstractFloatSearchScript.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+public abstract class AbstractFloatSearchScript extends AbstractSearchScript {
+
+    @Override public Object run() {
+        return runAsFloat();
+    }
+
+    @Override public abstract float runAsFloat();
+
+    @Override public double runAsDouble() {
+        return runAsFloat();
+    }
+
+    @Override public long runAsLong() {
+        return (long) runAsFloat();
+    }
+}

modules/elasticsearch/src/main/java/org/elasticsearch/script/AbstractLongSearchScript.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+public abstract class AbstractLongSearchScript extends AbstractSearchScript {
+
+    @Override public Object run() {
+        return runAsLong();
+    }
+
+    @Override public abstract long runAsLong();
+
+    @Override public double runAsDouble() {
+        return runAsLong();
+    }
+
+    @Override public float runAsFloat() {
+        return runAsLong();
+    }
+}

modules/elasticsearch/src/main/java/org/elasticsearch/script/AbstractSearchScript.java

@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Scorer;
+import org.elasticsearch.search.lookup.DocLookup;
+import org.elasticsearch.search.lookup.FieldsLookup;
+import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.search.lookup.SourceLookup;
+
+/**
+ * A base class for any script type that is used during the search process (custom score, facets, and so on).
+ *
+ * <p>If the script returns a specific numeric type, consider overriding the type specific base classes
+ * such as {@link AbstractDoubleSearchScript}, {@link AbstractFloatSearchScript} and {@link AbstractLongSearchScript}
+ * for better performance.
+ *
+ * <p>The use is required to implement the {@link #run()} method.
+ */
+public abstract class AbstractSearchScript extends AbstractExecutableScript implements SearchScript {
+
+    private SearchLookup lookup;
+
+    private float score = Float.NaN;
+
+    // helper methods
+    protected final float score() {
+        return score;
+    }
+
+    /**
+     * Returns the doc lookup allowing to access field data (cached) values as well as the current document score
+     * (where applicable).
+     */
+    protected final DocLookup doc() {
+        return lookup.doc();
+    }
+
+    /**
+     * Allows to access the actual source (loaded and parsed).
+     */
+    protected final SourceLookup source() {
+        return lookup.source();
+    }
+
+    /**
+     * Allows to access the *stored* fields.
+     */
+    protected final FieldsLookup fields() {
+        return lookup.fields();
+    }
+
+    void setLookup(SearchLookup lookup) {
+        this.lookup = lookup;
+    }
+
+    @Override public void setScorer(Scorer scorer) {
+        lookup.setScorer(scorer);
+    }
+
+    @Override public void setNextReader(IndexReader reader) {
+        lookup.setNextReader(reader);
+    }
+
+    @Override public void setNextDocId(int doc) {
+        lookup.setNextDocId(doc);
+    }
+
+    @Override public void setNextScore(float score) {
+        this.score = score;
+    }
+
+    @Override public float runAsFloat() {
+        return ((Number) run()).floatValue();
+    }
+
+    @Override public long runAsLong() {
+        return ((Number) run()).longValue();
+    }
+
+    @Override public double runAsDouble() {
+        return ((Number) run()).doubleValue();
+    }
+}

modules/elasticsearch/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java

@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.collect.ImmutableMap;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.search.lookup.SearchLookup;
+
+import java.util.Map;
+
+/**
+ * A native script engine service.
+ */
+public class NativeScriptEngineService extends AbstractComponent implements ScriptEngineService {
+
+    private final ImmutableMap<String, NativeScriptFactory> scripts;
+
+    @Inject public NativeScriptEngineService(Settings settings, Map<String, NativeScriptFactory> scripts) {
+        super(settings);
+        this.scripts = ImmutableMap.copyOf(scripts);
+    }
+
+    @Override public String[] types() {
+        return new String[]{"native"};
+    }
+
+    @Override public String[] extensions() {
+        return new String[0];
+    }
+
+    @Override public Object compile(String script) {
+        NativeScriptFactory scriptFactory = scripts.get(script);
+        if (scriptFactory != null) {
+            return scriptFactory;
+        }
+        throw new ElasticSearchIllegalArgumentException("Native script [" + script + "] not found");
+    }
+
+    @Override public ExecutableScript executable(Object compiledScript, @Nullable Map<String, Object> vars) {
+        NativeScriptFactory scriptFactory = (NativeScriptFactory) compiledScript;
+        return scriptFactory.newScript(vars);
+    }
+
+    @Override public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable Map<String, Object> vars) {
+        NativeScriptFactory scriptFactory = (NativeScriptFactory) compiledScript;
+        AbstractSearchScript script = (AbstractSearchScript) scriptFactory.newScript(vars);
+        script.setLookup(lookup);
+        return script;
+    }
+
+    @Override public Object execute(Object compiledScript, Map<String, Object> vars) {
+        return executable(compiledScript, vars).run();
+    }
+
+    @Override public Object unwrap(Object value) {
+        return value;
+    }
+
+    @Override public void close() {
+    }
+}

modules/elasticsearch/src/main/java/org/elasticsearch/script/NativeScriptFactory.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.common.Nullable;
+
+import java.util.Map;
+
+/**
+ * A factor to create instances of either {@link ExecutableScript} or {@link SearchScript}. Note,
+ * if this factor creates {@link SearchScript}, it must extend {@link AbstractSearchScript}.
+ *
+ * @see AbstractExecutableScript
+ * @see AbstractSearchScript
+ * @see AbstractFloatSearchScript
+ * @see AbstractLongSearchScript
+ * @see AbstractDoubleSearchScript
+ */
+public interface NativeScriptFactory {
+
+    /**
+     * Creates a new instance of either a {@link ExecutableScript} or a {@link SearchScript}.
+     *
+     * @param params The parameters passed to the script. Can be <tt>null</tt>.
+     */
+    ExecutableScript newScript(@Nullable Map<String, Object> params);
+}

modules/elasticsearch/src/main/java/org/elasticsearch/script/ScriptModule.java

@@ -19,26 +19,61 @@
 
 package org.elasticsearch.script;
 
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.collect.Lists;
+import org.elasticsearch.common.collect.Maps;
 import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.inject.multibindings.MapBinder;
 import org.elasticsearch.common.inject.multibindings.Multibinder;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.mvel.MvelScriptEngineService;
 
 import java.util.List;
+import java.util.Map;
 
 /**
  * @author kimchy (shay.banon)
  */
 public class ScriptModule extends AbstractModule {
 
-    private List<Class<? extends ScriptEngineService>> scriptEngines = Lists.newArrayList();
+    private final Settings settings;
+
+    private final List<Class<? extends ScriptEngineService>> scriptEngines = Lists.newArrayList();
+
+    private final Map<String, Class<? extends NativeScriptFactory>> scripts = Maps.newHashMap();
+
+    public ScriptModule(Settings settings) {
+        this.settings = settings;
+    }
 
     public void addScriptEngine(Class<? extends ScriptEngineService> scriptEngine) {
         scriptEngines.add(scriptEngine);
     }
 
+    public void registerScript(String name, Class<? extends NativeScriptFactory> script) {
+        scripts.put(name, script);
+    }
+
     @Override protected void configure() {
+        MapBinder<String, NativeScriptFactory> scriptsBinder
+                = MapBinder.newMapBinder(binder(), String.class, NativeScriptFactory.class);
+        for (Map.Entry<String, Class<? extends NativeScriptFactory>> entry : scripts.entrySet()) {
+            scriptsBinder.addBinding(entry.getKey()).to(entry.getValue());
+        }
+
+        // now, check for config based ones
+        Map<String, Settings> nativeSettings = settings.getGroups("script.native");
+        for (Map.Entry<String, Settings> entry : nativeSettings.entrySet()) {
+            String name = entry.getKey();
+            Class<? extends NativeScriptFactory> type = entry.getValue().getAsClass("type", NativeScriptFactory.class);
+            if (type == NativeScriptFactory.class) {
+                throw new ElasticSearchIllegalArgumentException("type is missing for native script [" + name + "]");
+            }
+            scriptsBinder.addBinding(name).to(type);
+        }
+
         Multibinder<ScriptEngineService> multibinder = Multibinder.newSetBinder(binder(), ScriptEngineService.class);
+        multibinder.addBinding().to(NativeScriptEngineService.class);
         try {
             multibinder.addBinding().to(MvelScriptEngineService.class);
         } catch (Throwable t) {

modules/elasticsearch/src/main/java/org/elasticsearch/script/SearchScript.java

@@ -25,7 +25,7 @@ import org.apache.lucene.search.Scorer;
 /**
  * A search script.
  */
-public interface SearchScript {
+public interface SearchScript extends ExecutableScript {
 
     void setScorer(Scorer scorer);
 
@@ -35,10 +35,6 @@ public interface SearchScript {
 
     void setNextScore(float score);
 
-    void setNextVar(String name, Object value);
-
-    Object run();
-
     float runAsFloat();
 
     long runAsLong();

modules/elasticsearch/src/main/java/org/elasticsearch/script/mvel/MvelScriptEngineService.java

@@ -179,5 +179,9 @@ public class MvelScriptEngineService extends AbstractComponent implements Script
         @Override public double runAsDouble() {
             return ((Number) run()).doubleValue();
         }
+
+        @Override public Object unwrap(Object value) {
+            return value;
+        }
     }
 }

modules/elasticsearch/src/test/java/org/elasticsearch/index/percolator/PercolatorExecutorTests.java

@@ -61,7 +61,7 @@ public class PercolatorExecutorTests {
         Injector injector = new ModulesBuilder().add(
                 new SettingsModule(settings),
                 new ThreadPoolModule(settings),
-                new ScriptModule(),
+                new ScriptModule(settings),
                 new MapperServiceModule(),
                 new IndexSettingsModule(settings),
                 new IndexCacheModule(settings),

modules/elasticsearch/src/test/java/org/elasticsearch/index/query/xcontent/SimpleIndexQueryParserTests.java

@@ -79,7 +79,7 @@ public class SimpleIndexQueryParserTests {
         Injector injector = new ModulesBuilder().add(
                 new SettingsModule(settings),
                 new ThreadPoolModule(settings),
-                new ScriptModule(),
+                new ScriptModule(settings),
                 new MapperServiceModule(),
                 new IndexSettingsModule(settings),
                 new IndexCacheModule(settings),

modules/elasticsearch/src/test/java/org/elasticsearch/index/query/xcontent/guice/IndexQueryParserModuleTests.java

@@ -59,7 +59,7 @@ public class IndexQueryParserModuleTests {
         Injector injector = new ModulesBuilder().add(
                 new SettingsModule(settings),
                 new ThreadPoolModule(settings),
-                new ScriptModule(),
+                new ScriptModule(settings),
                 new IndexSettingsModule(settings),
                 new IndexCacheModule(settings),
                 new AnalysisModule(settings),

modules/elasticsearch/src/test/java/org/elasticsearch/index/query/xcontent/plugin/IndexQueryParserPluginTests.java

@@ -64,7 +64,7 @@ public class IndexQueryParserPluginTests {
         Injector injector = new ModulesBuilder().add(
                 new SettingsModule(settings),
                 new ThreadPoolModule(settings),
-                new ScriptModule(),
+                new ScriptModule(settings),
                 new IndexSettingsModule(settings),
                 new IndexCacheModule(settings),
                 new AnalysisModule(settings),

modules/elasticsearch/src/test/java/org/elasticsearch/script/NativeScriptTests.java

@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.inject.Injector;
+import org.elasticsearch.common.inject.ModulesBuilder;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsModule;
+import org.testng.annotations.Test;
+
+import java.util.Map;
+
+import static org.hamcrest.MatcherAssert.*;
+import static org.hamcrest.Matchers.*;
+
+public class NativeScriptTests {
+
+    @Test public void testNativeScript() {
+        Settings settings = ImmutableSettings.settingsBuilder()
+                .put("script.native.my.type", MyNativeScriptFactory.class.getName())
+                .build();
+        Injector injector = new ModulesBuilder().add(
+                new SettingsModule(settings),
+                new ScriptModule(settings)).createInjector();
+
+        ScriptService scriptService = injector.getInstance(ScriptService.class);
+
+        ExecutableScript executable = scriptService.executable("native", "my", null);
+        assertThat(executable.run().toString(), equalTo("test"));
+    }
+
+    static class MyNativeScriptFactory implements NativeScriptFactory {
+        @Override public ExecutableScript newScript(@Nullable Map<String, Object> params) {
+            return new MyScript();
+        }
+    }
+
+    static class MyScript extends AbstractExecutableScript {
+        @Override public Object run() {
+            return "test";
+        }
+    }
+}

plugins/lang/groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java

@@ -186,5 +186,9 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
         @Override public double runAsDouble() {
             return ((Number) run()).doubleValue();
         }
+
+        @Override public Object unwrap(Object value) {
+            return value;
+        }
     }
 }

plugins/lang/javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java

@@ -243,6 +243,10 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements
         @Override public double runAsDouble() {
             return ((Number) run()).doubleValue();
         }
+
+        @Override public Object unwrap(Object value) {
+            return ScriptValueConverter.unwrapValue(value);
+        }
     }
 
     /**

plugins/lang/python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java

@@ -183,6 +183,10 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri
         @Override public double runAsDouble() {
             return ((Number) run()).doubleValue();
         }
+
+        @Override public Object unwrap(Object value) {
+            return unwrapValue(value);
+        }
     }