diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java index 7f420e6bb5d..7536f23d587 100644 --- a/core/src/main/java/org/elasticsearch/Version.java +++ b/core/src/main/java/org/elasticsearch/Version.java @@ -239,7 +239,9 @@ public class Version { public static final int V_1_5_3_ID = 1050399; public static final Version V_1_5_3 = new Version(V_1_5_3_ID, true, org.apache.lucene.util.Version.LUCENE_4_10_4); public static final int V_1_6_0_ID = 1060099; - public static final Version V_1_6_0 = new Version(V_1_6_0_ID, true, org.apache.lucene.util.Version.LUCENE_4_10_4); + public static final Version V_1_6_0 = new Version(V_1_6_0_ID, false, org.apache.lucene.util.Version.LUCENE_4_10_4); + public static final int V_1_6_1_ID = 1060199; + public static final Version V_1_6_1 = new Version(V_1_6_1_ID, true, org.apache.lucene.util.Version.LUCENE_4_10_4); public static final int V_2_0_0_ID = 2000099; public static final Version V_2_0_0 = new Version(V_2_0_0_ID, true, org.apache.lucene.util.Version.LUCENE_5_2_0); @@ -257,6 +259,8 @@ public class Version { switch (id) { case V_2_0_0_ID: return V_2_0_0; + case V_1_6_1_ID: + return V_1_6_1; case V_1_6_0_ID: return V_1_6_0; case V_1_5_3_ID: diff --git a/core/src/main/java/org/elasticsearch/common/component/AbstractComponent.java b/core/src/main/java/org/elasticsearch/common/component/AbstractComponent.java index a31bf119402..bed1f06676a 100644 --- a/core/src/main/java/org/elasticsearch/common/component/AbstractComponent.java +++ b/core/src/main/java/org/elasticsearch/common/component/AbstractComponent.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.component; +import com.google.common.base.Strings; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; @@ -51,4 +52,22 @@ public abstract class AbstractComponent { public final String nodeName() { return settings.get("name", ""); } + + /** + * Checks for a deprecated setting and logs the correct alternative + */ + protected void logDeprecatedSetting(String settingName, String alternativeName) { + if (!Strings.isNullOrEmpty(settings.get(settingName))) { + deprecationLogger.deprecated("Setting [{}] is deprecated, use [{}] instead", settingName, alternativeName); + } + } + + /** + * Checks for a removed setting and logs the correct alternative + */ + protected void logRemovedSetting(String settingName, String alternativeName) { + if (!Strings.isNullOrEmpty(settings.get(settingName))) { + deprecationLogger.deprecated("Setting [{}] has been removed, use [{}] instead", settingName, alternativeName); + } + } } diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java index 75ef6914eae..66be2de2750 100644 --- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -21,7 +21,9 @@ package org.elasticsearch.env; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; + import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.*; import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; @@ -116,11 +118,15 @@ public class NodeEnvironment extends AbstractComponent implements Closeable { // Setting to enable custom index.data_path setting for new indices public static final String SETTING_CUSTOM_DATA_PATH_ENABLED = "node.enable_custom_paths"; + // If enabled, the [verbose] SegmentInfos.infoStream logging is sent to System.out: + public static final String SETTING_ENABLE_LUCENE_SEGMENT_INFOS_TRACE = "node.enable_lucene_segment_infos_trace"; + public static final String NODES_FOLDER = "nodes"; public static final String INDICES_FOLDER = "indices"; public static final String NODE_LOCK_FILENAME = "node.lock"; @Inject + @SuppressForbidden(reason = "System.out.*") public NodeEnvironment(Settings settings, Environment environment) throws IOException { super(settings); @@ -186,6 +192,10 @@ public class NodeEnvironment extends AbstractComponent implements Closeable { } maybeLogPathDetails(); + + if (settings.getAsBoolean(SETTING_ENABLE_LUCENE_SEGMENT_INFOS_TRACE, false)) { + SegmentInfos.setInfoStream(System.out); + } } private static void releaseAndNullLocks(Lock[] locks) { diff --git a/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java index db343bf3555..02c2a17eade 100644 --- a/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; /** @@ -51,6 +52,14 @@ public class IdsQueryBuilder extends QueryBuilder implements BoostableQueryBuild return this; } + /** + * Adds ids to the filter. + */ + public IdsQueryBuilder addIds(Collection ids) { + values.addAll(ids); + return this; + } + /** * Adds ids to the filter. */ @@ -58,6 +67,13 @@ public class IdsQueryBuilder extends QueryBuilder implements BoostableQueryBuild return addIds(ids); } + /** + * Adds ids to the filter. + */ + public IdsQueryBuilder ids(Collection ids) { + return addIds(ids); + } + /** * Sets the boost for this query. Documents matching this query will (in addition to the normal * weightings) have their score multiplied by the boost provided. diff --git a/core/src/main/java/org/elasticsearch/watcher/ResourceWatcherService.java b/core/src/main/java/org/elasticsearch/watcher/ResourceWatcherService.java index ebb5bb84623..4719bd997d7 100644 --- a/core/src/main/java/org/elasticsearch/watcher/ResourceWatcherService.java +++ b/core/src/main/java/org/elasticsearch/watcher/ResourceWatcherService.java @@ -35,12 +35,12 @@ import java.util.concurrent.ScheduledFuture; * * Other elasticsearch services can register their resource watchers with this service using {@link #add(ResourceWatcher)} * method. This service will call {@link org.elasticsearch.watcher.ResourceWatcher#checkAndNotify()} method of all - * registered watcher periodically. The frequency of checks can be specified using {@code watcher.interval} setting, which - * defaults to {@code 60s}. The service can be disabled by setting {@code watcher.enabled} setting to {@code false}. + * registered watcher periodically. The frequency of checks can be specified using {@code resource.reload.interval} setting, which + * defaults to {@code 60s}. The service can be disabled by setting {@code resource.reload.enabled} setting to {@code false}. */ public class ResourceWatcherService extends AbstractLifecycleComponent { - public static enum Frequency { + public enum Frequency { /** * Defaults to 5 seconds @@ -59,7 +59,7 @@ public class ResourceWatcherService extends AbstractLifecycleComponent move shard from node_1 to node_3, and wait for relocation to finish"); + SlowClusterStateProcessing disruption = null; if (randomBoolean()) { // sometimes add cluster-state delay to trigger observers in IndicesStore.ShardActiveRequestHandler - final SlowClusterStateProcessing disruption = new SlowClusterStateProcessing(node_3, getRandom(), 0, 0, 1000, 2000); + disruption = new SlowClusterStateProcessing(node_3, getRandom(), 0, 0, 1000, 2000); internalCluster().setDisruptionScheme(disruption); disruption.startDisrupting(); } @@ -123,6 +124,12 @@ public class IndicesStoreIntegrationTests extends ElasticsearchIntegrationTest { .setWaitForRelocatingShards(0) .get(); assertThat(clusterHealth.isTimedOut(), equalTo(false)); + if (disruption != null) { + // we must stop the disruption here, else the delayed cluster state processing on the disrupted node + // can potentially delay registering the observer in IndicesStore.ShardActiveRequestHandler.messageReceived() + // and therefore sending the response for the shard active request for more than 10s + disruption.stopDisrupting(); + } assertThat(waitForShardDeletion(node_1, "test", 0), equalTo(false)); assertThat(waitForIndexDeletion(node_1, "test"), equalTo(false)); diff --git a/core/src/test/java/org/elasticsearch/watcher/ResourceWatcherServiceTests.java b/core/src/test/java/org/elasticsearch/watcher/ResourceWatcherServiceTests.java index 501289eadcd..ebb4cdaf768 100644 --- a/core/src/test/java/org/elasticsearch/watcher/ResourceWatcherServiceTests.java +++ b/core/src/test/java/org/elasticsearch/watcher/ResourceWatcherServiceTests.java @@ -45,7 +45,7 @@ public class ResourceWatcherServiceTests extends ElasticsearchTestCase { // checking bwc settings = Settings.builder() - .put("watcher.interval", "40s") // only applies to medium + .put("resource.reload.interval", "40s") // only applies to medium .build(); service = new ResourceWatcherService(settings, threadPool); assertThat(service.highMonitor.interval.millis(), is(timeValueSeconds(5).millis())); @@ -54,9 +54,9 @@ public class ResourceWatcherServiceTests extends ElasticsearchTestCase { // checking custom settings = Settings.builder() - .put("watcher.interval.high", "10s") - .put("watcher.interval.medium", "20s") - .put("watcher.interval.low", "30s") + .put("resource.reload.interval.high", "10s") + .put("resource.reload.interval.medium", "20s") + .put("resource.reload.interval.low", "30s") .build(); service = new ResourceWatcherService(settings, threadPool); assertThat(service.highMonitor.interval.millis(), is(timeValueSeconds(10).millis())); diff --git a/core/src/test/resources/org/elasticsearch/bwcompat/index-1.6.0.zip b/core/src/test/resources/org/elasticsearch/bwcompat/index-1.6.0.zip new file mode 100644 index 00000000000..02a5806638b Binary files /dev/null and b/core/src/test/resources/org/elasticsearch/bwcompat/index-1.6.0.zip differ diff --git a/core/src/test/resources/org/elasticsearch/bwcompat/repo-1.6.0.zip b/core/src/test/resources/org/elasticsearch/bwcompat/repo-1.6.0.zip new file mode 100644 index 00000000000..1c31a0240be Binary files /dev/null and b/core/src/test/resources/org/elasticsearch/bwcompat/repo-1.6.0.zip differ diff --git a/dev-tools/create_bwc_index.py b/dev-tools/create_bwc_index.py index 95f1acbad49..393b01606a9 100644 --- a/dev-tools/create_bwc_index.py +++ b/dev-tools/create_bwc_index.py @@ -130,7 +130,7 @@ def build_version(version_tuple): def build_tuple(version_string): return [int(x) for x in version_string.split('.')] -def start_node(version, release_dir, data_dir, tcp_port=DEFAULT_TRANSPORT_TCP_PORT, http_port=DEFAULT_HTTP_TCP_PORT, cluster_name=None): +def start_node(version, release_dir, data_dir, repo_dir, tcp_port=DEFAULT_TRANSPORT_TCP_PORT, http_port=DEFAULT_HTTP_TCP_PORT, cluster_name=None): logging.info('Starting node from %s on port %s/%s, data_dir %s' % (release_dir, tcp_port, http_port, data_dir)) if cluster_name is None: cluster_name = 'bwc_index_' + version @@ -143,7 +143,8 @@ def start_node(version, release_dir, data_dir, tcp_port=DEFAULT_TRANSPORT_TCP_PO '-Des.network.host=localhost', '-Des.discovery.zen.ping.multicast.enabled=false', '-Des.transport.tcp.port=%s' % tcp_port, - '-Des.http.port=%s' % http_port + '-Des.http.port=%s' % http_port, + '-Des.path.repo=%s' % repo_dir ] if version.startswith('0.') or version.startswith('1.0.0.Beta') : cmd.append('-f') # version before 1.0 start in background automatically @@ -329,7 +330,7 @@ def parse_config(): help='Recreate all existing backwards compatibility indexes') parser.add_argument('--releases-dir', '-d', default='backwards', metavar='DIR', help='The directory containing elasticsearch releases') - parser.add_argument('--output-dir', '-o', default='src/test/resources/org/elasticsearch/bwcompat', + parser.add_argument('--output-dir', '-o', default='core/src/test/resources/org/elasticsearch/bwcompat', help='The directory to write the zipped index into') parser.add_argument('--tcp-port', default=DEFAULT_TRANSPORT_TCP_PORT, type=int, help='The port to use as the minimum port for TCP communication') @@ -364,7 +365,7 @@ def create_bwc_index(cfg, version): node = None try: - node = start_node(version, release_dir, data_dir, cfg.tcp_port, cfg.http_port) + node = start_node(version, release_dir, data_dir, repo_dir, cfg.tcp_port, cfg.http_port) client = create_client(cfg.http_port) index_name = 'index-%s' % version.lower() generate_index(client, version, index_name) diff --git a/docs/reference/migration/migrate_2_0.asciidoc b/docs/reference/migration/migrate_2_0.asciidoc index eff219c5887..0965e53ba9d 100644 --- a/docs/reference/migration/migrate_2_0.asciidoc +++ b/docs/reference/migration/migrate_2_0.asciidoc @@ -685,3 +685,14 @@ curl -XGET 'localhost:9200/test/_search?fields=_timestamp,foo' } } --------------- + +=== Settings for resource watcher have been renamed + +The setting names for configuring the resource watcher have been renamed +to prevent clashes with the watcher plugin + +* `watcher.enabled` is now `resource.reload.enabled` +* `watcher.interval` is now `resource.reload.interval` +* `watcher.interval.low` is now `resource.reload.interval.low` +* `watcher.interval.medium` is now `resource.reload.interval.medium` +* `watcher.interval.high` is now `resource.reload.interval.high` diff --git a/docs/reference/modules/scripting.asciidoc b/docs/reference/modules/scripting.asciidoc index 0550542b4a2..3121388f43f 100644 --- a/docs/reference/modules/scripting.asciidoc +++ b/docs/reference/modules/scripting.asciidoc @@ -342,7 +342,7 @@ appropriate language. The `config/scripts` directory is scanned periodically for changes. New and changed scripts are reloaded and deleted script are removed from preloaded scripts cache. The reload frequency can be specified -using `watcher.interval` setting, which defaults to `60s`. +using `resource.reload.interval` setting, which defaults to `60s`. To disable script reloading completely set `script.auto_reload_enabled` to `false`. diff --git a/docs/reference/testing/testing-framework.asciidoc b/docs/reference/testing/testing-framework.asciidoc index a00484a4929..a3957c991ab 100644 --- a/docs/reference/testing/testing-framework.asciidoc +++ b/docs/reference/testing/testing-framework.asciidoc @@ -60,7 +60,7 @@ There are already have a couple of classes, you can inherit from in your own tes [[unit-tests]] === unit tests -In case you only need to execute a unit test, because your implementation can be isolated that good and does not require an up and running elasticsearch cluster, you can use the `ElasticsearchTestCase`. If you are testing lucene features, use `ElasticsearchLuceneTestCase` and if you are testing concrete token streams, use the `ElasticsearchTokenStreamTestCase` class. Those specific classes execute additional checks, which ensure that no resources leaks are happening, after the test has run. +In case you only need to execute a unit test, because your implementation can be isolated that well and does not require an up and running elasticsearch cluster, you can use the `ElasticsearchTestCase`. If you are testing lucene features, use `ElasticsearchLuceneTestCase` and if you are testing concrete token streams, use the `ElasticsearchTokenStreamTestCase` class. Those specific classes execute additional checks, which ensure that no resources leaks are happening, after the test has run. [[integration-tests]] diff --git a/migrate.sh b/migrate.sh index 44b808c1836..c8f80ccd828 100755 --- a/migrate.sh +++ b/migrate.sh @@ -13,11 +13,7 @@ set -o pipefail # ./migrate.sh # mvn clean install -DskipTests -DIR_TMP="../migration" -MODULE_CORE="core" -GIT_BRANCH="refactoring/maven" -PARENT_NAME="elasticsearch-parent" -PARENT_GIT="https://github.com/elastic/elasticsearch-parent.git" +GIT_BRANCH="refactoring/add_lang" # Insert a new text after a given line # insertLinesAfter text_to_find text_to_add newLine_separator filename @@ -105,11 +101,6 @@ function migratePlugin() { echo "# STEP 0 : prepare the job" -# echo "## clean $DIR_TMP plugins dev-tools/target target" -rm -rf $DIR_TMP -rm -rf plugins -rm -rf dev-tools/target -rm -rf target # echo "## create git $GIT_BRANCH work branch" @@ -121,99 +112,13 @@ git branch -D $GIT_BRANCH > /dev/null || : git branch $GIT_BRANCH > /dev/null git checkout $GIT_BRANCH > /dev/null 2>/dev/null -echo "# STEP 1 : Core module" - -# create the tmp work dir -# echo "## create $DIR_TMP temporary dir" -mkdir $DIR_TMP - -# create the core module -# echo "## create core module in $MODULE_CORE" -rm -rf $MODULE_CORE -mkdir $MODULE_CORE -# echo "## create $MODULE_CORE pom.xml" -cp pom.xml $MODULE_CORE -# echo "## modify $MODULE_CORE/pom.xml" -# We move block on top -removeLines "" "<\/parent>" "$MODULE_CORE/pom.xml" -insertLinesAfter "<\/modelVersion>" " § org.elasticsearch<\/groupId>§ elasticsearch-parent<\/artifactId>§ 2.0.0-SNAPSHOT<\/version>§ <\/parent>§" "§" "$MODULE_CORE/pom.xml" -# We clean useless data -replaceLine " 2.0.0-SNAPSHOT<\/version>" "" "$MODULE_CORE/pom.xml" -removeLines "" "<\/scm>" "$MODULE_CORE/pom.xml" -removeLines "" "<\/repositories>" "$MODULE_CORE/pom.xml" - -# echo "## move src in $MODULE_CORE" -git mv src/ $MODULE_CORE -# echo "## move bin in $MODULE_CORE" -git mv bin/ $MODULE_CORE -# echo "## move config in $MODULE_CORE" -git mv config/ $MODULE_CORE -# echo "## move lib in $MODULE_CORE" -git mv lib/ $MODULE_CORE -# echo "## copy README.textile, LICENSE.txt and NOTICE.txt in $MODULE_CORE" -cp README.textile $MODULE_CORE -cp LICENSE.txt $MODULE_CORE -cp NOTICE.txt $MODULE_CORE -# echo "## modify rest-api-spec location in $MODULE_CORE/pom.xml" -replaceLine " \${project.basedir}\/rest-api-spec<\/directory>" " \${project.basedir}\/..\/rest-api-spec<\/directory>" "$MODULE_CORE/pom.xml" - - -# echo "## commit changes" -git add . -git commit -m "create $MODULE_CORE module" > /dev/null - -echo "# STEP 2 : Parent pom.xml from $PARENT_GIT" - -# echo "## fetch parent project from $PARENT_GIT in $DIR_TMP" -# If you want to run that locally, uncomment this line and comment one below -# cp -R ../elasticsearch-parent $DIR_TMP -git clone $PARENT_GIT $DIR_TMP/$PARENT_NAME > /dev/null 2>/dev/null - -cp $DIR_TMP/$PARENT_NAME/pom.xml . -cp -R $DIR_TMP/$PARENT_NAME/dev-tools . -cp -R $DIR_TMP/$PARENT_NAME/plugins . - -# echo "## commit changes" -git add . -git commit -m "create parent pom project from its original location" > /dev/null - -echo "# STEP 3 : Add $MODULE_CORE module to pom.xml" - -insertLinesBefore " <\/modules>" " $MODULE_CORE<\/module>" "§" "pom.xml" - -# echo "## change name to Elasticsearch Core" -replaceLine " Elasticsearch core<\/name>" " Elasticsearch Core<\/name>" "$MODULE_CORE/pom.xml" - -# echo "## commit changes" -git add . -git commit -m "add core module" > /dev/null echo "# STEP 4 : Migrate plugins" -# We need to add in the plugins parent project as it does not exist -insertLinesBefore "<\/project>" " § <\/modules>" "§" "plugins/pom.xml" -git add plugins/pom.xml -git commit -m "add modules section" # Analysis -migratePlugin "analysis-kuromoji" -migratePlugin "analysis-smartcn" -migratePlugin "analysis-stempel" -migratePlugin "analysis-phonetic" -migratePlugin "analysis-icu" +migratePlugin "lang-python" +migratePlugin "lang-javascript" -# Mapper -# TODO: look at this one later -# migratePlugin "mapper-attachments" - -# Cloud -migratePlugin "cloud-gce" -migratePlugin "cloud-azure" -migratePlugin "cloud-aws" - -echo "# STEP 5 : Clean tmp dir" - -# echo "## clean $DIR_TMP" -rm -rf $DIR_TMP echo "you can now run:" echo "mvn clean install -DskipTests" diff --git a/plugins/lang-javascript/README.md b/plugins/lang-javascript/README.md new file mode 100644 index 00000000000..7080d9397d2 --- /dev/null +++ b/plugins/lang-javascript/README.md @@ -0,0 +1,177 @@ +JavaScript lang Plugin for Elasticsearch +================================== + +The JavaScript language plugin allows to have `javascript` (or `js`) as the language of scripts to execute. + +In order to install the plugin, simply run: + +```sh +bin/plugin install elasticsearch/elasticsearch-lang-javascript/2.5.0 +``` + +You need to install a version matching your Elasticsearch version: + +| elasticsearch | JavaScript Plugin | Docs | +|---------------|-----------------------|------------| +| master | Build from source | See below | +| es-1.x | Build from source | [2.6.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-transport-thrift/tree/es-1.x/#version-260-snapshot-for-elasticsearch-1x) | +| es-1.5 | 2.5.0 | [2.5.0](https://github.com/elastic/elasticsearch-lang-javascript/tree/v2.5.0/#version-250-for-elasticsearch-15) | +| es-1.4 | 2.4.1 | [2.4.1](https://github.com/elasticsearch/elasticsearch-lang-javascript/tree/v2.4.1/#version-241-for-elasticsearch-14) | +| es-1.3 | 2.3.1 | [2.3.1](https://github.com/elasticsearch/elasticsearch-lang-javascript/tree/v2.3.1/#version-231-for-elasticsearch-13) | +| es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-lang-javascript/tree/v2.2.0/#javascript-lang-plugin-for-elasticsearch) | +| es-1.1 | 2.1.0 | [2.1.0](https://github.com/elasticsearch/elasticsearch-lang-javascript/tree/v2.1.0/#javascript-lang-plugin-for-elasticsearch) | +| es-1.0 | 2.0.0 | [2.0.0](https://github.com/elasticsearch/elasticsearch-lang-javascript/tree/v2.0.0/#javascript-lang-plugin-for-elasticsearch) | +| es-0.90 | 1.4.0 | [1.4.0](https://github.com/elasticsearch/elasticsearch-lang-javascript/tree/v1.4.0/#javascript-lang-plugin-for-elasticsearch) | + +To build a `SNAPSHOT` version, you need to build it with Maven: + +```bash +mvn clean install +plugin --install lang-javascript \ + --url file:target/releases/elasticsearch-lang-javascript-X.X.X-SNAPSHOT.zip +``` + + +Using javascript with function_score +------------------------------------ + +Let's say you want to use `function_score` API using `javascript`. Here is +a way of doing it: + +```sh +curl -XDELETE "http://localhost:9200/test" + +curl -XPUT "http://localhost:9200/test/doc/1" -d '{ + "num": 1.0 +}' + +curl -XPUT "http://localhost:9200/test/doc/2?refresh" -d '{ + "num": 2.0 +}' + +curl -XGET "http://localhost:9200/test/_search?pretty" -d ' +{ + "query": { + "function_score": { + "script_score": { + "script": "doc[\"num\"].value", + "lang": "javascript" + } + } + } +}' +``` + +gives + +```javascript +{ + // ... + "hits": { + "total": 2, + "max_score": 4, + "hits": [ + { + // ... + "_score": 4 + }, + { + // ... + "_score": 1 + } + ] + } +} +``` + +Using javascript with script_fields +----------------------------------- + +```sh +curl -XDELETE "http://localhost:9200/test" + +curl -XPUT "http://localhost:9200/test/doc/1?refresh" -d' +{ + "obj1": { + "test": "something" + }, + "obj2": { + "arr2": [ "arr_value1", "arr_value2" ] + } +}' + +curl -XGET "http://localhost:9200/test/_search" -d' +{ + "script_fields": { + "s_obj1": { + "script": "_source.obj1", "lang": "js" + }, + "s_obj1_test": { + "script": "_source.obj1.test", "lang": "js" + }, + "s_obj2": { + "script": "_source.obj2", "lang": "js" + }, + "s_obj2_arr2": { + "script": "_source.obj2.arr2", "lang": "js" + } + } +}' +``` + +gives + +```javascript +{ + // ... + "hits": [ + { + // ... + "fields": { + "s_obj2_arr2": [ + [ + "arr_value1", + "arr_value2" + ] + ], + "s_obj1_test": [ + "something" + ], + "s_obj2": [ + { + "arr2": [ + "arr_value1", + "arr_value2" + ] + } + ], + "s_obj1": [ + { + "test": "something" + } + ] + } + } + ] +} +``` + + +License +------- + + This software is licensed under the Apache 2 license, quoted below. + + Copyright 2009-2014 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); you may not + use this file except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations under + the License. diff --git a/plugins/lang-javascript/pom.xml b/plugins/lang-javascript/pom.xml new file mode 100644 index 00000000000..7179582953c --- /dev/null +++ b/plugins/lang-javascript/pom.xml @@ -0,0 +1,43 @@ + + + 4.0.0 + + org.elasticsearch.plugin + elasticsearch-lang-javascript + + jar + Elasticsearch JavaScript language plugin + The JavaScript language plugin allows to have javascript as the language of scripts to execute. + + + org.elasticsearch + elasticsearch-plugin + 2.0.0-SNAPSHOT + + + + + + + + + org.mozilla + rhino + 1.7R4 + + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + + + diff --git a/plugins/lang-javascript/src/main/assemblies/plugin.xml b/plugins/lang-javascript/src/main/assemblies/plugin.xml new file mode 100644 index 00000000000..cc9a1cb3189 --- /dev/null +++ b/plugins/lang-javascript/src/main/assemblies/plugin.xml @@ -0,0 +1,26 @@ + + + plugin + + zip + + false + + + / + true + true + + org.elasticsearch:elasticsearch + + + + / + true + true + + org.mozilla:rhino + + + + \ No newline at end of file diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java new file mode 100644 index 00000000000..0b4ffd7fae5 --- /dev/null +++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java @@ -0,0 +1,44 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugin.javascript; + +import org.elasticsearch.plugins.AbstractPlugin; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.javascript.JavaScriptScriptEngineService; + +/** + * + */ +public class JavaScriptPlugin extends AbstractPlugin { + + @Override + public String name() { + return "lang-javascript"; + } + + @Override + public String description() { + return "JavaScript plugin allowing to add javascript scripting support"; + } + + public void onModule(ScriptModule module) { + module.addScriptEngine(JavaScriptScriptEngineService.class); + } +} diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java new file mode 100644 index 00000000000..7aa8b2bbf3f --- /dev/null +++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java @@ -0,0 +1,311 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.javascript; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Scorer; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.*; +import org.elasticsearch.script.javascript.support.NativeList; +import org.elasticsearch.script.javascript.support.NativeMap; +import org.elasticsearch.script.javascript.support.ScriptValueConverter; +import org.elasticsearch.search.lookup.LeafSearchLookup; +import org.elasticsearch.search.lookup.SearchLookup; +import org.mozilla.javascript.*; +import org.mozilla.javascript.Script; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** + * + */ +public class JavaScriptScriptEngineService extends AbstractComponent implements ScriptEngineService { + + private final AtomicLong counter = new AtomicLong(); + + private static WrapFactory wrapFactory = new CustomWrapFactory(); + + private final int optimizationLevel; + + private Scriptable globalScope; + + @Inject + public JavaScriptScriptEngineService(Settings settings) { + super(settings); + + this.optimizationLevel = settings.getAsInt("script.javascript.optimization_level", 1); + + Context ctx = Context.enter(); + try { + ctx.setWrapFactory(wrapFactory); + globalScope = ctx.initStandardObjects(null, true); + } finally { + Context.exit(); + } + } + + @Override + public void close() { + + } + + @Override + public void scriptRemoved(@Nullable CompiledScript compiledScript) { + // Nothing to do here + } + + @Override + public String[] types() { + return new String[]{"js", "javascript"}; + } + + @Override + public String[] extensions() { + return new String[]{"js"}; + } + + @Override + public boolean sandboxed() { + return false; + } + + @Override + public Object compile(String script) { + Context ctx = Context.enter(); + try { + ctx.setWrapFactory(wrapFactory); + ctx.setOptimizationLevel(optimizationLevel); + return ctx.compileString(script, generateScriptName(), 1, null); + } finally { + Context.exit(); + } + } + + @Override + public ExecutableScript executable(Object compiledScript, Map vars) { + Context ctx = Context.enter(); + try { + ctx.setWrapFactory(wrapFactory); + + Scriptable scope = ctx.newObject(globalScope); + scope.setPrototype(globalScope); + scope.setParentScope(null); + for (Map.Entry entry : vars.entrySet()) { + ScriptableObject.putProperty(scope, entry.getKey(), entry.getValue()); + } + + return new JavaScriptExecutableScript((Script) compiledScript, scope); + } finally { + Context.exit(); + } + } + + @Override + public SearchScript search(final Object compiledScript, final SearchLookup lookup, @Nullable final Map vars) { + Context ctx = Context.enter(); + try { + ctx.setWrapFactory(wrapFactory); + + final Scriptable scope = ctx.newObject(globalScope); + scope.setPrototype(globalScope); + scope.setParentScope(null); + + return new SearchScript() { + + @Override + public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException { + final LeafSearchLookup leafLookup = lookup.getLeafSearchLookup(context); + for (Map.Entry entry : leafLookup.asMap().entrySet()) { + ScriptableObject.putProperty(scope, entry.getKey(), entry.getValue()); + } + + if (vars != null) { + for (Map.Entry entry : vars.entrySet()) { + ScriptableObject.putProperty(scope, entry.getKey(), entry.getValue()); + } + } + + return new JavaScriptSearchScript((Script) compiledScript, scope, leafLookup); + } + }; + } finally { + Context.exit(); + } + } + + @Override + public Object execute(Object compiledScript, Map vars) { + Context ctx = Context.enter(); + ctx.setWrapFactory(wrapFactory); + try { + Script script = (Script) compiledScript; + Scriptable scope = ctx.newObject(globalScope); + scope.setPrototype(globalScope); + scope.setParentScope(null); + + for (Map.Entry entry : vars.entrySet()) { + ScriptableObject.putProperty(scope, entry.getKey(), entry.getValue()); + } + Object ret = script.exec(ctx, scope); + return ScriptValueConverter.unwrapValue(ret); + } finally { + Context.exit(); + } + } + + @Override + public Object unwrap(Object value) { + return ScriptValueConverter.unwrapValue(value); + } + + private String generateScriptName() { + return "Script" + counter.incrementAndGet() + ".js"; + } + + public static class JavaScriptExecutableScript implements ExecutableScript { + + private final Script script; + + private final Scriptable scope; + + public JavaScriptExecutableScript(Script script, Scriptable scope) { + this.script = script; + this.scope = scope; + } + + @Override + public Object run() { + Context ctx = Context.enter(); + try { + ctx.setWrapFactory(wrapFactory); + return ScriptValueConverter.unwrapValue(script.exec(ctx, scope)); + } finally { + Context.exit(); + } + } + + @Override + public void setNextVar(String name, Object value) { + ScriptableObject.putProperty(scope, name, value); + } + + @Override + public Object unwrap(Object value) { + return ScriptValueConverter.unwrapValue(value); + } + } + + public static class JavaScriptSearchScript implements LeafSearchScript { + + private final Script script; + + private final Scriptable scope; + + private final LeafSearchLookup lookup; + + public JavaScriptSearchScript(Script script, Scriptable scope, LeafSearchLookup lookup) { + this.script = script; + this.scope = scope; + this.lookup = lookup; + } + + @Override + public void setScorer(Scorer scorer) { + Context ctx = Context.enter(); + try { + ScriptableObject.putProperty(scope, "_score", wrapFactory.wrapAsJavaObject(ctx, scope, new ScoreAccessor(scorer), ScoreAccessor.class)); + } finally { + Context.exit(); + } + } + + @Override + public void setDocument(int doc) { + lookup.setDocument(doc); + } + + @Override + public void setNextVar(String name, Object value) { + ScriptableObject.putProperty(scope, name, value); + } + + @Override + public void setSource(Map source) { + lookup.source().setSource(source); + } + + @Override + public Object run() { + Context ctx = Context.enter(); + try { + ctx.setWrapFactory(wrapFactory); + return ScriptValueConverter.unwrapValue(script.exec(ctx, scope)); + } finally { + Context.exit(); + } + } + + @Override + public float runAsFloat() { + return ((Number) run()).floatValue(); + } + + @Override + public long runAsLong() { + return ((Number) run()).longValue(); + } + + @Override + public double runAsDouble() { + return ((Number) run()).doubleValue(); + } + + @Override + public Object unwrap(Object value) { + return ScriptValueConverter.unwrapValue(value); + } + } + + /** + * Wrap Factory for Rhino Script Engine + */ + public static class CustomWrapFactory extends WrapFactory { + + public CustomWrapFactory() { + setJavaPrimitiveWrap(false); // RingoJS does that..., claims its annoying... + } + + public Scriptable wrapAsJavaObject(Context cx, Scriptable scope, Object javaObject, Class staticType) { + if (javaObject instanceof Map) { + return NativeMap.wrap(scope, (Map) javaObject); + } + if (javaObject instanceof List) { + return NativeList.wrap(scope, (List) javaObject, staticType); + } + return super.wrapAsJavaObject(cx, scope, javaObject, staticType); + } + } +} diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/NativeList.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/NativeList.java new file mode 100644 index 00000000000..1bf555db81e --- /dev/null +++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/NativeList.java @@ -0,0 +1,149 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.javascript.support; + +import org.mozilla.javascript.NativeJavaObject; +import org.mozilla.javascript.Scriptable; +import org.mozilla.javascript.Undefined; +import org.mozilla.javascript.Wrapper; + +import java.util.Arrays; +import java.util.List; + +/** + * + */ +public class NativeList extends NativeJavaObject implements Scriptable, Wrapper { + private static final long serialVersionUID = 3664761893203964569L; + private static final String LENGTH_PROPERTY = "length"; + + private final List list; + + + public static NativeList wrap(Scriptable scope, List list, Class staticType) { + return new NativeList(scope, list, staticType); + } + + private NativeList(Scriptable scope, List list, Class staticType) { + super(scope, list, staticType); + this.list = list; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Wrapper#unwrap() + */ + + public Object unwrap() { + return list; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#getClassName() + */ + + public String getClassName() { + return "NativeList"; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#get(java.lang.String, org.mozilla.javascript.Scriptable) + */ + + public Object get(String name, Scriptable start) { + if (LENGTH_PROPERTY.equals(name)) { + return list.size(); + } else { + return super.get(name, start); + } + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#get(int, org.mozilla.javascript.Scriptable) + */ + + public Object get(int index, Scriptable start) { + if (has(index, start) == false) { + return Undefined.instance; + } + return list.get(index); + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#has(java.lang.String, org.mozilla.javascript.Scriptable) + */ + + public boolean has(String name, Scriptable start) { + return super.has(name, start) || LENGTH_PROPERTY.equals(name); + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#has(int, org.mozilla.javascript.Scriptable) + */ + + public boolean has(int index, Scriptable start) { + return index >= 0 && index < list.size(); + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#put(int, org.mozilla.javascript.Scriptable, java.lang.Object) + */ + + public void put(int index, Scriptable start, Object value) { + if (index == list.size()) { + list.add(value); + } else { + list.set(index, value); + } + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#delete(int) + */ + + public void delete(int index) { + list.remove(index); + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#getIds() + */ + + public Object[] getIds() { + final Object[] javaObjectIds = super.getIds(); + final int size = list.size(); + final Object[] ids = Arrays.copyOf(javaObjectIds, javaObjectIds.length + size); + for (int i = 0; i < size; ++i) { + ids[javaObjectIds.length + i] = i; + } + return ids; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#hasInstance(org.mozilla.javascript.Scriptable) + */ + + public boolean hasInstance(Scriptable value) { + if (!(value instanceof Wrapper)) + return false; + Object instance = ((Wrapper) value).unwrap(); + return List.class.isInstance(instance); + } + +} diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/NativeMap.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/NativeMap.java new file mode 100644 index 00000000000..a43ecec0ec8 --- /dev/null +++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/NativeMap.java @@ -0,0 +1,223 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.javascript.support; + +import org.mozilla.javascript.Scriptable; +import org.mozilla.javascript.Wrapper; + +import java.util.Iterator; +import java.util.Map; + +/** + * Wrapper for exposing maps in Rhino scripts. + * + * + */ +public class NativeMap implements Scriptable, Wrapper { + private static final long serialVersionUID = 3664761893203964569L; + + private Map map; + private Scriptable parentScope; + private Scriptable prototype; + + + /** + * Construct + * + * @param scope + * @param map + * @return native map + */ + public static NativeMap wrap(Scriptable scope, Map map) { + return new NativeMap(scope, map); + } + + /** + * Construct + * + * @param scope + * @param map + */ + private NativeMap(Scriptable scope, Map map) { + this.parentScope = scope; + this.map = map; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Wrapper#unwrap() + */ + + public Object unwrap() { + return map; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#getClassName() + */ + + public String getClassName() { + return "NativeMap"; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#get(java.lang.String, org.mozilla.javascript.Scriptable) + */ + + public Object get(String name, Scriptable start) { + // get the property from the underlying QName map + if ("length".equals(name)) { + return map.size(); + } else { + return map.get(name); + } + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#get(int, org.mozilla.javascript.Scriptable) + */ + + public Object get(int index, Scriptable start) { + Object value = null; + int i = 0; + Iterator itrValues = map.values().iterator(); + while (i++ <= index && itrValues.hasNext()) { + value = itrValues.next(); + } + return value; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#has(java.lang.String, org.mozilla.javascript.Scriptable) + */ + + public boolean has(String name, Scriptable start) { + // locate the property in the underlying map + return map.containsKey(name); + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#has(int, org.mozilla.javascript.Scriptable) + */ + + public boolean has(int index, Scriptable start) { + return (index >= 0 && map.values().size() > index); + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#put(java.lang.String, org.mozilla.javascript.Scriptable, java.lang.Object) + */ + + @SuppressWarnings("unchecked") + public void put(String name, Scriptable start, Object value) { + map.put(name, value); + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#put(int, org.mozilla.javascript.Scriptable, java.lang.Object) + */ + + public void put(int index, Scriptable start, Object value) { + // TODO: implement? + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#delete(java.lang.String) + */ + + public void delete(String name) { + map.remove(name); + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#delete(int) + */ + + public void delete(int index) { + int i = 0; + Iterator itrKeys = map.keySet().iterator(); + while (i <= index && itrKeys.hasNext()) { + Object key = itrKeys.next(); + if (i == index) { + map.remove(key); + break; + } + } + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#getPrototype() + */ + + public Scriptable getPrototype() { + return this.prototype; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#setPrototype(org.mozilla.javascript.Scriptable) + */ + + public void setPrototype(Scriptable prototype) { + this.prototype = prototype; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#getParentScope() + */ + + public Scriptable getParentScope() { + return this.parentScope; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#setParentScope(org.mozilla.javascript.Scriptable) + */ + + public void setParentScope(Scriptable parent) { + this.parentScope = parent; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#getIds() + */ + + public Object[] getIds() { + return map.keySet().toArray(); + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#getDefaultValue(java.lang.Class) + */ + + public Object getDefaultValue(Class hint) { + return null; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#hasInstance(org.mozilla.javascript.Scriptable) + */ + + public boolean hasInstance(Scriptable value) { + if (!(value instanceof Wrapper)) + return false; + Object instance = ((Wrapper) value).unwrap(); + return Map.class.isInstance(instance); + } + +} diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/ScriptValueConverter.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/ScriptValueConverter.java new file mode 100644 index 00000000000..7f9c390b270 --- /dev/null +++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/ScriptValueConverter.java @@ -0,0 +1,183 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.javascript.support; + +import org.mozilla.javascript.*; + +import java.util.*; + +/** + * Value Converter to marshal objects between Java and Javascript. + * + * + */ +public final class ScriptValueConverter { + private static final String TYPE_DATE = "Date"; + + + /** + * Private constructor - methods are static + */ + private ScriptValueConverter() { + } + + /** + * Convert an object from a script wrapper value to a serializable value valid outside + * of the Rhino script processor context. + *

+ * This includes converting JavaScript Array objects to Lists of valid objects. + * + * @param value Value to convert from script wrapper object to external object value. + * @return unwrapped and converted value. + */ + public static Object unwrapValue(Object value) { + if (value == null) { + return null; + } else if (value instanceof Wrapper) { + // unwrap a Java object from a JavaScript wrapper + // recursively call this method to convert the unwrapped value + value = unwrapValue(((Wrapper) value).unwrap()); + } else if (value instanceof IdScriptableObject) { + // check for special case Native object wrappers + String className = ((IdScriptableObject) value).getClassName(); + // check for special case of the String object + if ("String".equals(className)) { + value = Context.jsToJava(value, String.class); + } + // check for special case of a Date object + else if ("Date".equals(className)) { + value = Context.jsToJava(value, Date.class); + } else { + // a scriptable object will probably indicate a multi-value property set + // set using a JavaScript associative Array object + Scriptable values = (Scriptable) value; + Object[] propIds = values.getIds(); + + // is it a JavaScript associative Array object using Integer indexes? + if (values instanceof NativeArray && isArray(propIds)) { + // convert JavaScript array of values to a List of Serializable objects + List propValues = new ArrayList(propIds.length); + for (int i = 0; i < propIds.length; i++) { + // work on each key in turn + Integer propId = (Integer) propIds[i]; + + // we are only interested in keys that indicate a list of values + if (propId instanceof Integer) { + // get the value out for the specified key + Object val = values.get(propId, values); + // recursively call this method to convert the value + propValues.add(unwrapValue(val)); + } + } + + value = propValues; + } else { + // any other JavaScript object that supports properties - convert to a Map of objects + Map propValues = new HashMap(propIds.length); + for (int i = 0; i < propIds.length; i++) { + // work on each key in turn + Object propId = propIds[i]; + + // we are only interested in keys that indicate a list of values + if (propId instanceof String) { + // get the value out for the specified key + Object val = values.get((String) propId, values); + // recursively call this method to convert the value + propValues.put((String) propId, unwrapValue(val)); + } + } + value = propValues; + } + } + } else if (value instanceof Object[]) { + // convert back a list Object Java values + Object[] array = (Object[]) value; + ArrayList list = new ArrayList(array.length); + for (int i = 0; i < array.length; i++) { + list.add(unwrapValue(array[i])); + } + value = list; + } else if (value instanceof Map) { + // ensure each value in the Map is unwrapped (which may have been an unwrapped NativeMap!) + Map map = (Map) value; + Map copyMap = new HashMap(map.size()); + for (Object key : map.keySet()) { + copyMap.put(key, unwrapValue(map.get(key))); + } + value = copyMap; + } + return value; + } + + /** + * Convert an object from any repository serialized value to a valid script object. + * This includes converting Collection multi-value properties into JavaScript Array objects. + * + * @param scope Scripting scope + * @param value Property value + * @return Value safe for scripting usage + */ + public static Object wrapValue(Scriptable scope, Object value) { + // perform conversions from Java objects to JavaScript scriptable instances + if (value == null) { + return null; + } else if (value instanceof Date) { + // convert Date to JavaScript native Date object + // call the "Date" constructor on the root scope object - passing in the millisecond + // value from the Java date - this will construct a JavaScript Date with the same value + Date date = (Date) value; + value = ScriptRuntime.newObject( + Context.getCurrentContext(), scope, TYPE_DATE, new Object[]{date.getTime()}); + } else if (value instanceof Collection) { + // recursively convert each value in the collection + Collection collection = (Collection) value; + Object[] array = new Object[collection.size()]; + int index = 0; + for (Object obj : collection) { + array[index++] = wrapValue(scope, obj); + } + // convert array to a native JavaScript Array + value = Context.getCurrentContext().newArray(scope, array); + } else if (value instanceof Map) { + value = NativeMap.wrap(scope, (Map) value); + } + + // simple numbers, strings and booleans are wrapped automatically by Rhino + + return value; + } + + /** + * Look at the id's of a native array and try to determine whether it's actually an Array or a Hashmap + * + * @param ids id's of the native array + * @return boolean true if it's an array, false otherwise (ie it's a map) + */ + private static boolean isArray(final Object[] ids) { + boolean result = true; + for (int i = 0; i < ids.length; i++) { + if (ids[i] instanceof Integer == false) { + result = false; + break; + } + } + return result; + } +} diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/ScriptableLinkedHashMap.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/ScriptableLinkedHashMap.java new file mode 100644 index 00000000000..680b20a0256 --- /dev/null +++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/ScriptableLinkedHashMap.java @@ -0,0 +1,188 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.javascript.support; + +import org.mozilla.javascript.Scriptable; + +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Implementation of a Scriptable Map. This is the best choice for maps that want to represent + * JavaScript associative arrays - allowing access via key and integer index. It maintains and + * respects insertion order of the elements and allows either string or integer keys. + * + * + */ +public class ScriptableLinkedHashMap extends LinkedHashMap implements ScriptableMap { + private static final long serialVersionUID = 3774167893214964123L; + + private Scriptable parentScope; + private Scriptable prototype; + + + public ScriptableLinkedHashMap() { + } + + public ScriptableLinkedHashMap(int initialCapacity) { + super(initialCapacity); + } + + public ScriptableLinkedHashMap(Map source) { + super(source); + } + + /** + * @see org.mozilla.javascript.Scriptable#getClassName() + */ + public String getClassName() { + return "ScriptableMap"; + } + + /** + * @see org.mozilla.javascript.Scriptable#get(java.lang.String, org.mozilla.javascript.Scriptable) + */ + public Object get(String name, Scriptable start) { + // get the property from the underlying QName map + if ("length".equals(name)) { + return this.size(); + } else { + return get(name); + } + } + + /** + * @see org.mozilla.javascript.Scriptable#get(int, org.mozilla.javascript.Scriptable) + */ + public Object get(int index, Scriptable start) { + Object value = null; + int i = 0; + Iterator itrValues = this.values().iterator(); + while (i++ <= index && itrValues.hasNext()) { + value = itrValues.next(); + } + return value; + } + + /** + * @see org.mozilla.javascript.Scriptable#has(java.lang.String, org.mozilla.javascript.Scriptable) + */ + public boolean has(String name, Scriptable start) { + // locate the property in the underlying map + return containsKey(name); + } + + /** + * @see org.mozilla.javascript.Scriptable#has(int, org.mozilla.javascript.Scriptable) + */ + public boolean has(int index, Scriptable start) { + return (index >= 0 && this.values().size() > index); + } + + /** + * @see org.mozilla.javascript.Scriptable#put(java.lang.String, org.mozilla.javascript.Scriptable, java.lang.Object) + */ + @SuppressWarnings("unchecked") + public void put(String name, Scriptable start, Object value) { + // add the property to the underlying QName map + put((K) name, (V) value); + } + + /** + * @see org.mozilla.javascript.Scriptable#put(int, org.mozilla.javascript.Scriptable, java.lang.Object) + */ + public void put(int index, Scriptable start, Object value) { + // TODO: implement? + } + + /** + * @see org.mozilla.javascript.Scriptable#delete(java.lang.String) + */ + public void delete(String name) { + // remove the property from the underlying QName map + remove(name); + } + + /** + * @see org.mozilla.javascript.Scriptable#delete(int) + */ + public void delete(int index) { + int i = 0; + Iterator itrKeys = this.keySet().iterator(); + while (i <= index && itrKeys.hasNext()) { + Object key = itrKeys.next(); + if (i == index) { + remove(key); + break; + } + } + } + + /** + * @see org.mozilla.javascript.Scriptable#getPrototype() + */ + public Scriptable getPrototype() { + return this.prototype; + } + + /** + * @see org.mozilla.javascript.Scriptable#setPrototype(org.mozilla.javascript.Scriptable) + */ + public void setPrototype(Scriptable prototype) { + this.prototype = prototype; + } + + /** + * @see org.mozilla.javascript.Scriptable#getParentScope() + */ + public Scriptable getParentScope() { + return this.parentScope; + } + + /** + * @see org.mozilla.javascript.Scriptable#setParentScope(org.mozilla.javascript.Scriptable) + */ + public void setParentScope(Scriptable parent) { + this.parentScope = parent; + } + + /** + * @see org.mozilla.javascript.Scriptable#getIds() + */ + public Object[] getIds() { + return keySet().toArray(); + } + + /** + * @see org.mozilla.javascript.Scriptable#getDefaultValue(java.lang.Class) + */ + public Object getDefaultValue(Class hint) { + return null; + } + + /** + * @see org.mozilla.javascript.Scriptable#hasInstance(org.mozilla.javascript.Scriptable) + */ + public boolean hasInstance(Scriptable instance) { + return instance instanceof ScriptableLinkedHashMap; + } +} + diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/ScriptableMap.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/ScriptableMap.java new file mode 100644 index 00000000000..ed910c5afca --- /dev/null +++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/ScriptableMap.java @@ -0,0 +1,32 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.javascript.support; + +import org.mozilla.javascript.Scriptable; + +import java.util.Map; + +/** + * Contract to be implemented by classes providing Map like collections to JavaScript. + * + * + */ +public interface ScriptableMap extends Scriptable, Map { +} diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/ScriptableWrappedMap.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/ScriptableWrappedMap.java new file mode 100644 index 00000000000..a7c2f4f5428 --- /dev/null +++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/support/ScriptableWrappedMap.java @@ -0,0 +1,342 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.javascript.support; + +import org.mozilla.javascript.Scriptable; +import org.mozilla.javascript.Wrapper; + +import java.util.Collection; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +/** + * Implementation of a Scriptable Map. This is the best choice where you want values to be + * persisted directly to an underlying map supplied on construction. The class automatically + * wraps/unwraps JS objects as they enter/leave the underlying map via the Scriptable interface + * methods - objects are untouched if accessed via the usual Map interface methods. + *

+ *

Access should be by string key only - not integer index - unless you are sure the wrapped + * map will maintain insertion order of the elements. + * + * + */ +public class ScriptableWrappedMap implements ScriptableMap, Wrapper { + private Map map; + private Scriptable parentScope; + private Scriptable prototype; + + + /** + * Construction + * + * @param scope + * @param map + * @return scriptable wrapped map + */ + public static ScriptableWrappedMap wrap(Scriptable scope, Map map) { + return new ScriptableWrappedMap(scope, map); + } + + /** + * Construct + * + * @param map + */ + public ScriptableWrappedMap(Map map) { + this.map = map; + } + + /** + * Construct + * + * @param scope + * @param map + */ + public ScriptableWrappedMap(Scriptable scope, Map map) { + this.parentScope = scope; + this.map = map; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Wrapper#unwrap() + */ + + public Object unwrap() { + return map; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#getClassName() + */ + + public String getClassName() { + return "ScriptableWrappedMap"; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#get(java.lang.String, org.mozilla.javascript.Scriptable) + */ + + public Object get(String name, Scriptable start) { + // get the property from the underlying QName map + if ("length".equals(name)) { + return map.size(); + } else { + return ScriptValueConverter.wrapValue(this.parentScope != null ? this.parentScope : start, map.get(name)); + } + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#get(int, org.mozilla.javascript.Scriptable) + */ + + public Object get(int index, Scriptable start) { + Object value = null; + int i = 0; + Iterator itrValues = map.values().iterator(); + while (i++ <= index && itrValues.hasNext()) { + value = itrValues.next(); + } + return ScriptValueConverter.wrapValue(this.parentScope != null ? this.parentScope : start, value); + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#has(java.lang.String, org.mozilla.javascript.Scriptable) + */ + + public boolean has(String name, Scriptable start) { + // locate the property in the underlying map + return map.containsKey(name); + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#has(int, org.mozilla.javascript.Scriptable) + */ + + public boolean has(int index, Scriptable start) { + return (index >= 0 && map.values().size() > index); + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#put(java.lang.String, org.mozilla.javascript.Scriptable, java.lang.Object) + */ + + @SuppressWarnings("unchecked") + public void put(String name, Scriptable start, Object value) { + map.put(name, ScriptValueConverter.unwrapValue(value)); + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#put(int, org.mozilla.javascript.Scriptable, java.lang.Object) + */ + + public void put(int index, Scriptable start, Object value) { + // TODO: implement? + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#delete(java.lang.String) + */ + + public void delete(String name) { + map.remove(name); + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#delete(int) + */ + + public void delete(int index) { + int i = 0; + Iterator itrKeys = map.keySet().iterator(); + while (i <= index && itrKeys.hasNext()) { + Object key = itrKeys.next(); + if (i == index) { + map.remove(key); + break; + } + } + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#getPrototype() + */ + + public Scriptable getPrototype() { + return this.prototype; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#setPrototype(org.mozilla.javascript.Scriptable) + */ + + public void setPrototype(Scriptable prototype) { + this.prototype = prototype; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#getParentScope() + */ + + public Scriptable getParentScope() { + return this.parentScope; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#setParentScope(org.mozilla.javascript.Scriptable) + */ + + public void setParentScope(Scriptable parent) { + this.parentScope = parent; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#getIds() + */ + + public Object[] getIds() { + return map.keySet().toArray(); + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#getDefaultValue(java.lang.Class) + */ + + public Object getDefaultValue(Class hint) { + return null; + } + + /* (non-Javadoc) + * @see org.mozilla.javascript.Scriptable#hasInstance(org.mozilla.javascript.Scriptable) + */ + + public boolean hasInstance(Scriptable value) { + if (!(value instanceof Wrapper)) + return false; + Object instance = ((Wrapper) value).unwrap(); + return Map.class.isInstance(instance); + } + + /* (non-Javadoc) + * @see java.util.Map#clear() + */ + + public void clear() { + this.map.clear(); + } + + /* (non-Javadoc) + * @see java.util.Map#containsKey(java.lang.Object) + */ + + public boolean containsKey(Object key) { + return this.map.containsKey(key); + } + + /* (non-Javadoc) + * @see java.util.Map#containsValue(java.lang.Object) + */ + + public boolean containsValue(Object value) { + return this.map.containsValue(value); + } + + /* (non-Javadoc) + * @see java.util.Map#entrySet() + */ + + public Set entrySet() { + return this.map.entrySet(); + } + + /* (non-Javadoc) + * @see java.util.Map#get(java.lang.Object) + */ + + public Object get(Object key) { + return this.map.get(key); + } + + /* (non-Javadoc) + * @see java.util.Map#isEmpty() + */ + + public boolean isEmpty() { + return (this.map.size() == 0); + } + + /* (non-Javadoc) + * @see java.util.Map#keySet() + */ + + public Set keySet() { + return this.map.keySet(); + } + + /* (non-Javadoc) + * @see java.util.Map#put(java.lang.Object, java.lang.Object) + */ + + public Object put(Object key, Object value) { + return this.map.put(key, value); + } + + /* (non-Javadoc) + * @see java.util.Map#putAll(java.util.Map) + */ + + public void putAll(Map t) { + this.map.putAll(t); + } + + /* (non-Javadoc) + * @see java.util.Map#remove(java.lang.Object) + */ + + public Object remove(Object key) { + return this.map.remove(key); + } + + /* (non-Javadoc) + * @see java.util.Map#size() + */ + + public int size() { + return this.map.size(); + } + + /* (non-Javadoc) + * @see java.util.Map#values() + */ + + public Collection values() { + return this.map.values(); + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + + @Override + public String toString() { + return (this.map != null ? this.map.toString() : super.toString()); + } +} diff --git a/plugins/lang-javascript/src/main/resources/es-plugin.properties b/plugins/lang-javascript/src/main/resources/es-plugin.properties new file mode 100644 index 00000000000..92f26b778d7 --- /dev/null +++ b/plugins/lang-javascript/src/main/resources/es-plugin.properties @@ -0,0 +1,2 @@ +plugin=org.elasticsearch.plugin.javascript.JavaScriptPlugin +version=${project.version} diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineTests.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineTests.java new file mode 100644 index 00000000000..9bb3543b012 --- /dev/null +++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineTests.java @@ -0,0 +1,174 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.javascript; + +import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.test.ElasticsearchTestCase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +/** + * + */ +public class JavaScriptScriptEngineTests extends ElasticsearchTestCase { + + private JavaScriptScriptEngineService se; + + @Before + public void setup() { + se = new JavaScriptScriptEngineService(Settings.Builder.EMPTY_SETTINGS); + } + + @After + public void close() { + se.close(); + } + + @Test + public void testSimpleEquation() { + Map vars = new HashMap(); + Object o = se.execute(se.compile("1 + 2"), vars); + assertThat(((Number) o).intValue(), equalTo(3)); + } + + @Test + public void testMapAccess() { + Map vars = new HashMap(); + + Map obj2 = MapBuilder.newMapBuilder().put("prop2", "value2").map(); + Map obj1 = MapBuilder.newMapBuilder().put("prop1", "value1").put("obj2", obj2).put("l", Arrays.asList("2", "1")).map(); + vars.put("obj1", obj1); + Object o = se.execute(se.compile("obj1"), vars); + assertThat(o, instanceOf(Map.class)); + obj1 = (Map) o; + assertThat((String) obj1.get("prop1"), equalTo("value1")); + assertThat((String) ((Map) obj1.get("obj2")).get("prop2"), equalTo("value2")); + + o = se.execute(se.compile("obj1.l[0]"), vars); + assertThat(((String) o), equalTo("2")); + } + + @Test + public void testJavaScriptObjectToMap() { + Map vars = new HashMap(); + Object o = se.execute(se.compile("var obj1 = {}; obj1.prop1 = 'value1'; obj1.obj2 = {}; obj1.obj2.prop2 = 'value2'; obj1"), vars); + Map obj1 = (Map) o; + assertThat((String) obj1.get("prop1"), equalTo("value1")); + assertThat((String) ((Map) obj1.get("obj2")).get("prop2"), equalTo("value2")); + } + + @Test + public void testJavaScriptObjectMapInter() { + Map vars = new HashMap(); + Map ctx = new HashMap(); + Map obj1 = new HashMap(); + obj1.put("prop1", "value1"); + ctx.put("obj1", obj1); + vars.put("ctx", ctx); + + se.execute(se.compile("ctx.obj2 = {}; ctx.obj2.prop2 = 'value2'; ctx.obj1.prop1 = 'uvalue1'"), vars); + ctx = (Map) se.unwrap(vars.get("ctx")); + assertThat(ctx.containsKey("obj1"), equalTo(true)); + assertThat((String) ((Map) ctx.get("obj1")).get("prop1"), equalTo("uvalue1")); + assertThat(ctx.containsKey("obj2"), equalTo(true)); + assertThat((String) ((Map) ctx.get("obj2")).get("prop2"), equalTo("value2")); + } + + @Test + public void testJavaScriptInnerArrayCreation() { + Map ctx = new HashMap(); + Map doc = new HashMap(); + ctx.put("doc", doc); + + Object complied = se.compile("ctx.doc.field1 = ['value1', 'value2']"); + ExecutableScript script = se.executable(complied, new HashMap()); + script.setNextVar("ctx", ctx); + script.run(); + + Map unwrap = (Map) script.unwrap(ctx); + + assertThat(((Map) unwrap.get("doc")).get("field1"), instanceOf(List.class)); + } + + @Test + public void testAccessListInScript() { + Map vars = new HashMap(); + Map obj2 = MapBuilder.newMapBuilder().put("prop2", "value2").map(); + Map obj1 = MapBuilder.newMapBuilder().put("prop1", "value1").put("obj2", obj2).map(); + vars.put("l", Arrays.asList("1", "2", "3", obj1)); + + Object o = se.execute(se.compile("l.length"), vars); + assertThat(((Number) o).intValue(), equalTo(4)); + + o = se.execute(se.compile("l[0]"), vars); + assertThat(((String) o), equalTo("1")); + + o = se.execute(se.compile("l[3]"), vars); + obj1 = (Map) o; + assertThat((String) obj1.get("prop1"), equalTo("value1")); + assertThat((String) ((Map) obj1.get("obj2")).get("prop2"), equalTo("value2")); + + o = se.execute(se.compile("l[3].prop1"), vars); + assertThat(((String) o), equalTo("value1")); + } + + @Test + public void testChangingVarsCrossExecution1() { + Map vars = new HashMap(); + Map ctx = new HashMap(); + vars.put("ctx", ctx); + Object compiledScript = se.compile("ctx.value"); + + ExecutableScript script = se.executable(compiledScript, vars); + ctx.put("value", 1); + Object o = script.run(); + assertThat(((Number) o).intValue(), equalTo(1)); + + ctx.put("value", 2); + o = script.run(); + assertThat(((Number) o).intValue(), equalTo(2)); + } + + @Test + public void testChangingVarsCrossExecution2() { + Map vars = new HashMap(); + Object compiledScript = se.compile("value"); + + ExecutableScript script = se.executable(compiledScript, vars); + script.setNextVar("value", 1); + Object o = script.run(); + assertThat(((Number) o).intValue(), equalTo(1)); + + script.setNextVar("value", 2); + o = script.run(); + assertThat(((Number) o).intValue(), equalTo(2)); + } +} diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTest.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTest.java new file mode 100644 index 00000000000..c235128e83f --- /dev/null +++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTest.java @@ -0,0 +1,169 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.javascript; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.test.ElasticsearchTestCase; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.hamcrest.Matchers.equalTo; + +/** + * + */ +public class JavaScriptScriptMultiThreadedTest extends ElasticsearchTestCase { + + @Test + public void testExecutableNoRuntimeParams() throws Exception { + final JavaScriptScriptEngineService se = new JavaScriptScriptEngineService(Settings.Builder.EMPTY_SETTINGS); + final Object compiled = se.compile("x + y"); + final AtomicBoolean failed = new AtomicBoolean(); + + Thread[] threads = new Thread[50]; + final CountDownLatch latch = new CountDownLatch(threads.length); + final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1); + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(new Runnable() { + @Override + public void run() { + try { + barrier.await(); + long x = ThreadLocalRandom.current().nextInt(); + long y = ThreadLocalRandom.current().nextInt(); + long addition = x + y; + Map vars = new HashMap(); + vars.put("x", x); + vars.put("y", y); + ExecutableScript script = se.executable(compiled, vars); + for (int i = 0; i < 100000; i++) { + long result = ((Number) script.run()).longValue(); + assertThat(result, equalTo(addition)); + } + } catch (Throwable t) { + failed.set(true); + logger.error("failed", t); + } finally { + latch.countDown(); + } + } + }); + } + for (int i = 0; i < threads.length; i++) { + threads[i].start(); + } + barrier.await(); + latch.await(); + assertThat(failed.get(), equalTo(false)); + } + + + @Test + public void testExecutableWithRuntimeParams() throws Exception { + final JavaScriptScriptEngineService se = new JavaScriptScriptEngineService(Settings.Builder.EMPTY_SETTINGS); + final Object compiled = se.compile("x + y"); + final AtomicBoolean failed = new AtomicBoolean(); + + Thread[] threads = new Thread[50]; + final CountDownLatch latch = new CountDownLatch(threads.length); + final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1); + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(new Runnable() { + @Override + public void run() { + try { + barrier.await(); + long x = ThreadLocalRandom.current().nextInt(); + Map vars = new HashMap(); + vars.put("x", x); + ExecutableScript script = se.executable(compiled, vars); + for (int i = 0; i < 100000; i++) { + long y = ThreadLocalRandom.current().nextInt(); + long addition = x + y; + script.setNextVar("y", y); + long result = ((Number) script.run()).longValue(); + assertThat(result, equalTo(addition)); + } + } catch (Throwable t) { + failed.set(true); + logger.error("failed", t); + } finally { + latch.countDown(); + } + } + }); + } + for (int i = 0; i < threads.length; i++) { + threads[i].start(); + } + barrier.await(); + latch.await(); + assertThat(failed.get(), equalTo(false)); + } + + @Test + public void testExecute() throws Exception { + final JavaScriptScriptEngineService se = new JavaScriptScriptEngineService(Settings.Builder.EMPTY_SETTINGS); + final Object compiled = se.compile("x + y"); + final AtomicBoolean failed = new AtomicBoolean(); + + Thread[] threads = new Thread[50]; + final CountDownLatch latch = new CountDownLatch(threads.length); + final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1); + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(new Runnable() { + @Override + public void run() { + try { + barrier.await(); + Map runtimeVars = new HashMap(); + for (int i = 0; i < 100000; i++) { + long x = ThreadLocalRandom.current().nextInt(); + long y = ThreadLocalRandom.current().nextInt(); + long addition = x + y; + runtimeVars.put("x", x); + runtimeVars.put("y", y); + long result = ((Number) se.execute(compiled, runtimeVars)).longValue(); + assertThat(result, equalTo(addition)); + } + } catch (Throwable t) { + failed.set(true); + logger.error("failed", t); + } finally { + latch.countDown(); + } + } + }); + } + for (int i = 0; i < threads.length; i++) { + threads[i].start(); + } + barrier.await(); + latch.await(); + assertThat(failed.get(), equalTo(false)); + } +} diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptSearchTests.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptSearchTests.java new file mode 100644 index 00000000000..53cf8d198da --- /dev/null +++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptSearchTests.java @@ -0,0 +1,299 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.javascript; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.client.Requests.searchRequest; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.QueryBuilders.*; +import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; +import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +/** + * + */ +@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.SUITE) +public class JavaScriptScriptSearchTests extends ElasticsearchIntegrationTest { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .build(); + } + + @Test + public void testJavaScriptFilter() throws Exception { + createIndex("test"); + index("test", "type1", "1", jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject()); + flush(); + index("test", "type1", "2", jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject()); + flush(); + index("test", "type1", "3", jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).endObject()); + refresh(); + + logger.info(" --> running doc['num1'].value > 1"); + SearchResponse response = client().prepareSearch() + .setQuery(filteredQuery(matchAllQuery(), scriptQuery("doc['num1'].value > 1").lang("js"))) + .addSort("num1", SortOrder.ASC) + .addScriptField("sNum1", "js", "doc['num1'].value", null) + .execute().actionGet(); + + assertThat(response.getHits().totalHits(), equalTo(2l)); + assertThat(response.getHits().getAt(0).id(), equalTo("2")); + assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0)); + assertThat(response.getHits().getAt(1).id(), equalTo("3")); + assertThat((Double) response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(3.0)); + + logger.info(" --> running doc['num1'].value > param1"); + response = client().prepareSearch() + .setQuery(filteredQuery(matchAllQuery(), scriptQuery("doc['num1'].value > param1").lang("js").addParam("param1", 2))) + .addSort("num1", SortOrder.ASC) + .addScriptField("sNum1", "js", "doc['num1'].value", null) + .execute().actionGet(); + + assertThat(response.getHits().totalHits(), equalTo(1l)); + assertThat(response.getHits().getAt(0).id(), equalTo("3")); + assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(3.0)); + + logger.info(" --> running doc['num1'].value > param1"); + response = client().prepareSearch() + .setQuery(filteredQuery(matchAllQuery(), scriptQuery("doc['num1'].value > param1").lang("js").addParam("param1", -1))) + .addSort("num1", SortOrder.ASC) + .addScriptField("sNum1", "js", "doc['num1'].value", null) + .execute().actionGet(); + + assertThat(response.getHits().totalHits(), equalTo(3l)); + assertThat(response.getHits().getAt(0).id(), equalTo("1")); + assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0)); + assertThat(response.getHits().getAt(1).id(), equalTo("2")); + assertThat((Double) response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0)); + assertThat(response.getHits().getAt(2).id(), equalTo("3")); + assertThat((Double) response.getHits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0)); + } + + @Test + public void testScriptFieldUsingSource() throws Exception { + createIndex("test"); + index("test", "type1", "1", + jsonBuilder().startObject() + .startObject("obj1").field("test", "something").endObject() + .startObject("obj2").startArray("arr2").value("arr_value1").value("arr_value2").endArray().endObject() + .endObject()); + refresh(); + + SearchResponse response = client().prepareSearch() + .setQuery(matchAllQuery()) + .addScriptField("s_obj1", "js", "_source.obj1", null) + .addScriptField("s_obj1_test", "js", "_source.obj1.test", null) + .addScriptField("s_obj2", "js", "_source.obj2", null) + .addScriptField("s_obj2_arr2", "js", "_source.obj2.arr2", null) + .execute().actionGet(); + + Map sObj1 = (Map) response.getHits().getAt(0).field("s_obj1").value(); + assertThat(sObj1.get("test").toString(), equalTo("something")); + assertThat(response.getHits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something")); + + Map sObj2 = (Map) response.getHits().getAt(0).field("s_obj2").value(); + List sObj2Arr2 = (List) sObj2.get("arr2"); + assertThat(sObj2Arr2.size(), equalTo(2)); + assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1")); + assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2")); + + sObj2Arr2 = (List) response.getHits().getAt(0).field("s_obj2_arr2").values(); + assertThat(sObj2Arr2.size(), equalTo(2)); + assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1")); + assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2")); + } + + @Test + public void testCustomScriptBoost() throws Exception { + createIndex("test"); + index("test", "type1", "1", jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject()); + index("test", "type1", "2", jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject()); + refresh(); + + logger.info("--- QUERY_THEN_FETCH"); + + logger.info(" --> running doc['num1'].value"); + SearchResponse response = client().search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("doc['num1'].value").lang("js")))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); + + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + assertThat(response.getHits().getAt(0).id(), equalTo("2")); + assertThat(response.getHits().getAt(1).id(), equalTo("1")); + + logger.info(" --> running -doc['num1'].value"); + response = client().search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("-doc['num1'].value").lang("js")))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); + + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + assertThat(response.getHits().getAt(0).id(), equalTo("1")); + assertThat(response.getHits().getAt(1).id(), equalTo("2")); + + + logger.info(" --> running pow(doc['num1'].value, 2)"); + response = client().search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("Math.pow(doc['num1'].value, 2)").lang("js")))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); + + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + assertThat(response.getHits().getAt(0).id(), equalTo("2")); + assertThat(response.getHits().getAt(1).id(), equalTo("1")); + + logger.info(" --> running max(doc['num1'].value, 1)"); + response = client().search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("Math.max(doc['num1'].value, 1)").lang("js")))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); + + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + assertThat(response.getHits().getAt(0).id(), equalTo("2")); + assertThat(response.getHits().getAt(1).id(), equalTo("1")); + + logger.info(" --> running doc['num1'].value * _score"); + response = client().search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("doc['num1'].value * _score").lang("js")))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); + + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + assertThat(response.getHits().getAt(0).id(), equalTo("2")); + assertThat(response.getHits().getAt(1).id(), equalTo("1")); + + logger.info(" --> running param1 * param2 * _score"); + response = client().search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("param1 * param2 * _score").param("param1", 2).param("param2", 2).lang("js")))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); + + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + } + + @Test + public void testScriptScoresNested() throws IOException { + createIndex("index"); + ensureYellow(); + index("index", "testtype", "1", jsonBuilder().startObject().field("dummy_field", 1).endObject()); + refresh(); + SearchResponse response = client().search( + searchRequest().source( + searchSource().query( + functionScoreQuery( + functionScoreQuery( + functionScoreQuery().add(scriptFunction("1").lang("js"))) + .add(scriptFunction("_score.doubleValue()").lang("js"))) + .add(scriptFunction("_score.doubleValue()").lang("js") + ) + ) + ) + ).actionGet(); + assertSearchResponse(response); + assertThat(response.getHits().getAt(0).score(), equalTo(1.0f)); + } + + @Test + public void testScriptScoresWithAgg() throws IOException { + createIndex("index"); + ensureYellow(); + index("index", "testtype", "1", jsonBuilder().startObject().field("dummy_field", 1).endObject()); + refresh(); + SearchResponse response = client().search( + searchRequest().source( + searchSource().query( + functionScoreQuery() + .add(scriptFunction("_score.doubleValue()").lang("js") + ) + ).aggregation(terms("score_agg").script("_score.doubleValue()").lang("js")) + ) + ).actionGet(); + assertSearchResponse(response); + assertThat(response.getHits().getAt(0).score(), equalTo(1.0f)); + assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getKeyAsNumber().floatValue(), is(1f)); + assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getDocCount(), is(1l)); + } + + @Test + public void testUseListLengthInScripts() throws Exception { + createIndex("index"); + index("index", "testtype", "1", jsonBuilder().startObject().field("f", 42).endObject()); + ensureSearchable("index"); + refresh(); + SearchResponse response = client().prepareSearch().addScriptField("foobar", "js", "doc['f'].values.length", null).get(); + assertSearchResponse(response); + assertHitCount(response, 1); + assertThat((Integer) response.getHits().getAt(0).getFields().get("foobar").value(), equalTo(1)); + } +} diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/SimpleBench.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/SimpleBench.java new file mode 100644 index 00000000000..e0b47c8a919 --- /dev/null +++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/SimpleBench.java @@ -0,0 +1,71 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.javascript; + +import org.elasticsearch.common.StopWatch; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ExecutableScript; + +import java.util.HashMap; +import java.util.Map; + +/** + * + */ +public class SimpleBench { + + public static void main(String[] args) { + JavaScriptScriptEngineService se = new JavaScriptScriptEngineService(Settings.Builder.EMPTY_SETTINGS); + Object compiled = se.compile("x + y"); + + Map vars = new HashMap(); + // warm up + for (int i = 0; i < 1000; i++) { + vars.put("x", i); + vars.put("y", i + 1); + se.execute(compiled, vars); + } + + final long ITER = 100000; + + StopWatch stopWatch = new StopWatch().start(); + for (long i = 0; i < ITER; i++) { + se.execute(compiled, vars); + } + System.out.println("Execute Took: " + stopWatch.stop().lastTaskTime()); + + stopWatch = new StopWatch().start(); + ExecutableScript executableScript = se.executable(compiled, vars); + for (long i = 0; i < ITER; i++) { + executableScript.run(); + } + System.out.println("Executable Took: " + stopWatch.stop().lastTaskTime()); + + stopWatch = new StopWatch().start(); + executableScript = se.executable(compiled, vars); + for (long i = 0; i < ITER; i++) { + for (Map.Entry entry : vars.entrySet()) { + executableScript.setNextVar(entry.getKey(), entry.getValue()); + } + executableScript.run(); + } + System.out.println("Executable (vars) Took: " + stopWatch.stop().lastTaskTime()); + } +} diff --git a/plugins/lang-python/README.md b/plugins/lang-python/README.md new file mode 100644 index 00000000000..6e6116b1028 --- /dev/null +++ b/plugins/lang-python/README.md @@ -0,0 +1,178 @@ +Python lang Plugin for Elasticsearch +================================== + +The Python (jython) language plugin allows to have `python` as the language of scripts to execute. + +In order to install the plugin, simply run: + +```sh +bin/plugin install elasticsearch/elasticsearch-lang-python/2.5.0 +``` + +You need to install a version matching your Elasticsearch version: + +| elasticsearch | Python Lang Plugin | Docs | +|---------------|-----------------------|------------| +| master | Build from source | See below | +| es-1.x | Build from source | [2.6.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-lang-python/tree/es-1.x/#version-260-snapshot-for-elasticsearch-1x) | +| es-1.5 | 2.5.0 | [2.5.0](https://github.com/elastic/elasticsearch-lang-python/tree/v2.5.0/#version-250-for-elasticsearch-15) | +| es-1.4 | 2.4.1 | [2.4.1](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.4.1/#version-241-for-elasticsearch-14) | +| es-1.3 | 2.3.1 | [2.3.1](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.1/#version-231-for-elasticsearch-13) | +| < 1.3.5 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.3.0/#version-230-for-elasticsearch-13) | +| es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.2.0/#python-lang-plugin-for-elasticsearch) | +| es-1.0 | 2.0.0 | [2.0.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v2.0.0/#python-lang-plugin-for-elasticsearch) | +| es-0.90 | 1.0.0 | [1.0.0](https://github.com/elasticsearch/elasticsearch-lang-python/tree/v1.0.0/#python-lang-plugin-for-elasticsearch) | + +To build a `SNAPSHOT` version, you need to build it with Maven: + +```bash +mvn clean install +plugin --install lang-python \ + --url file:target/releases/elasticsearch-lang-python-X.X.X-SNAPSHOT.zip +``` + +User Guide +---------- + +Using python with function_score +-------------------------------- + +Let's say you want to use `function_score` API using `python`. Here is +a way of doing it: + +```sh +curl -XDELETE "http://localhost:9200/test" + +curl -XPUT "http://localhost:9200/test/doc/1" -d '{ + "num": 1.0 +}' + +curl -XPUT "http://localhost:9200/test/doc/2?refresh" -d '{ + "num": 2.0 +}' + +curl -XGET "http://localhost:9200/test/_search?pretty" -d' +{ + "query": { + "function_score": { + "script_score": { + "script": "doc[\"num\"].value * _score", + "lang": "python" + } + } + } +}' +``` + +gives + +```javascript +{ + // ... + "hits": { + "total": 2, + "max_score": 2, + "hits": [ + { + // ... + "_score": 2 + }, + { + // ... + "_score": 1 + } + ] + } +} +``` + +Using python with script_fields +------------------------------- + +```sh +curl -XDELETE "http://localhost:9200/test" + +curl -XPUT "http://localhost:9200/test/doc/1?refresh" -d' +{ + "obj1": { + "test": "something" + }, + "obj2": { + "arr2": [ "arr_value1", "arr_value2" ] + } +}' + +curl -XGET "http://localhost:9200/test/_search" -d' +{ + "script_fields": { + "s_obj1": { + "script": "_source[\"obj1\"]", "lang": "python" + }, + "s_obj1_test": { + "script": "_source[\"obj1\"][\"test\"]", "lang": "python" + }, + "s_obj2": { + "script": "_source[\"obj2\"]", "lang": "python" + }, + "s_obj2_arr2": { + "script": "_source[\"obj2\"][\"arr2\"]", "lang": "python" + } + } +}' +``` + +gives + +```javascript +{ + // ... + "hits": [ + { + // ... + "fields": { + "s_obj2_arr2": [ + [ + "arr_value1", + "arr_value2" + ] + ], + "s_obj1_test": [ + "something" + ], + "s_obj2": [ + { + "arr2": [ + "arr_value1", + "arr_value2" + ] + } + ], + "s_obj1": [ + { + "test": "something" + } + ] + } + } + ] +} +``` + +License +------- + + This software is licensed under the Apache 2 license, quoted below. + + Copyright 2009-2014 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); you may not + use this file except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations under + the License. diff --git a/plugins/lang-python/pom.xml b/plugins/lang-python/pom.xml new file mode 100644 index 00000000000..1d52498628c --- /dev/null +++ b/plugins/lang-python/pom.xml @@ -0,0 +1,42 @@ + + + 4.0.0 + + org.elasticsearch.plugin + elasticsearch-lang-python + + jar + Elasticsearch Python language plugin + The Python language plugin allows to have python as the language of scripts to execute. + + + org.elasticsearch + elasticsearch-plugin + 2.0.0-SNAPSHOT + + + + + + + + + + org.python + jython-standalone + 2.7.0 + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + + + diff --git a/plugins/lang-python/src/main/assemblies/plugin.xml b/plugins/lang-python/src/main/assemblies/plugin.xml new file mode 100644 index 00000000000..037ea9f7ee8 --- /dev/null +++ b/plugins/lang-python/src/main/assemblies/plugin.xml @@ -0,0 +1,26 @@ + + + plugin + + zip + + false + + + / + true + true + + org.elasticsearch:elasticsearch + + + + / + true + true + + org.python:jython-standalone + + + + \ No newline at end of file diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java b/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java new file mode 100644 index 00000000000..78f05311a6f --- /dev/null +++ b/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java @@ -0,0 +1,44 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugin.python; + +import org.elasticsearch.plugins.AbstractPlugin; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.python.PythonScriptEngineService; + +/** + * + */ +public class PythonPlugin extends AbstractPlugin { + + @Override + public String name() { + return "lang-python"; + } + + @Override + public String description() { + return "Adds support for writing scripts in Python"; + } + + public void onModule(ScriptModule module) { + module.addScriptEngine(PythonScriptEngineService.class); + } +} diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java new file mode 100644 index 00000000000..6138453925e --- /dev/null +++ b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java @@ -0,0 +1,242 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.python; + +import java.io.IOException; +import java.util.Map; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Scorer; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.LeafSearchScript; +import org.elasticsearch.script.ScoreAccessor; +import org.elasticsearch.script.ScriptEngineService; +import org.elasticsearch.script.SearchScript; +import org.elasticsearch.search.lookup.LeafSearchLookup; +import org.elasticsearch.search.lookup.SearchLookup; +import org.python.core.Py; +import org.python.core.PyCode; +import org.python.core.PyObject; +import org.python.core.PyStringMap; +import org.python.util.PythonInterpreter; + +/** + * + */ +//TODO we can optimize the case for Map similar to PyStringMap +public class PythonScriptEngineService extends AbstractComponent implements ScriptEngineService { + + private final PythonInterpreter interp; + + @Inject + public PythonScriptEngineService(Settings settings) { + super(settings); + + this.interp = PythonInterpreter.threadLocalStateInterpreter(null); + } + + @Override + public String[] types() { + return new String[]{"python", "py"}; + } + + @Override + public String[] extensions() { + return new String[]{"py"}; + } + + @Override + public boolean sandboxed() { + return false; + } + + @Override + public Object compile(String script) { + return interp.compile(script); + } + + @Override + public ExecutableScript executable(Object compiledScript, Map vars) { + return new PythonExecutableScript((PyCode) compiledScript, vars); + } + + @Override + public SearchScript search(final Object compiledScript, final SearchLookup lookup, @Nullable final Map vars) { + return new SearchScript() { + @Override + public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException { + final LeafSearchLookup leafLookup = lookup.getLeafSearchLookup(context); + return new PythonSearchScript((PyCode) compiledScript, vars, leafLookup); + } + }; + } + + @Override + public Object execute(Object compiledScript, Map vars) { + PyObject pyVars = Py.java2py(vars); + interp.setLocals(pyVars); + PyObject ret = interp.eval((PyCode) compiledScript); + if (ret == null) { + return null; + } + return ret.__tojava__(Object.class); + } + + @Override + public Object unwrap(Object value) { + return unwrapValue(value); + } + + @Override + public void close() { + interp.cleanup(); + } + + @Override + public void scriptRemoved(@Nullable CompiledScript compiledScript) { + // Nothing to do + } + + public class PythonExecutableScript implements ExecutableScript { + + private final PyCode code; + + private final PyStringMap pyVars; + + public PythonExecutableScript(PyCode code, Map vars) { + this.code = code; + this.pyVars = new PyStringMap(); + if (vars != null) { + for (Map.Entry entry : vars.entrySet()) { + pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue())); + } + } + } + + @Override + public void setNextVar(String name, Object value) { + pyVars.__setitem__(name, Py.java2py(value)); + } + + @Override + public Object run() { + interp.setLocals(pyVars); + PyObject ret = interp.eval(code); + if (ret == null) { + return null; + } + return ret.__tojava__(Object.class); + } + + @Override + public Object unwrap(Object value) { + return unwrapValue(value); + } + } + + public class PythonSearchScript implements LeafSearchScript { + + private final PyCode code; + + private final PyStringMap pyVars; + + private final LeafSearchLookup lookup; + + public PythonSearchScript(PyCode code, Map vars, LeafSearchLookup lookup) { + this.code = code; + this.pyVars = new PyStringMap(); + for (Map.Entry entry : lookup.asMap().entrySet()) { + pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue())); + } + if (vars != null) { + for (Map.Entry entry : vars.entrySet()) { + pyVars.__setitem__(entry.getKey(), Py.java2py(entry.getValue())); + } + } + this.lookup = lookup; + } + + @Override + public void setScorer(Scorer scorer) { + pyVars.__setitem__("_score", Py.java2py(new ScoreAccessor(scorer))); + } + + @Override + public void setDocument(int doc) { + lookup.setDocument(doc); + } + + @Override + public void setSource(Map source) { + lookup.source().setSource(source); + } + + @Override + public void setNextVar(String name, Object value) { + pyVars.__setitem__(name, Py.java2py(value)); + } + + @Override + public Object run() { + interp.setLocals(pyVars); + PyObject ret = interp.eval(code); + if (ret == null) { + return null; + } + return ret.__tojava__(Object.class); + } + + @Override + public float runAsFloat() { + return ((Number) run()).floatValue(); + } + + @Override + public long runAsLong() { + return ((Number) run()).longValue(); + } + + @Override + public double runAsDouble() { + return ((Number) run()).doubleValue(); + } + + @Override + public Object unwrap(Object value) { + return unwrapValue(value); + } + } + + + public static Object unwrapValue(Object value) { + if (value == null) { + return null; + } else if (value instanceof PyObject) { + // seems like this is enough, inner PyDictionary will do the conversion for us for example, so expose it directly + return ((PyObject) value).__tojava__(Object.class); + } + return value; + } +} diff --git a/plugins/lang-python/src/main/resources/es-plugin.properties b/plugins/lang-python/src/main/resources/es-plugin.properties new file mode 100644 index 00000000000..b0ed0d5e3e0 --- /dev/null +++ b/plugins/lang-python/src/main/resources/es-plugin.properties @@ -0,0 +1,2 @@ +plugin=org.elasticsearch.plugin.python.PythonPlugin +version=${project.version} diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java new file mode 100644 index 00000000000..1621d22ac01 --- /dev/null +++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java @@ -0,0 +1,153 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.python; + +import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.test.ElasticsearchTestCase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +/** + * + */ +public class PythonScriptEngineTests extends ElasticsearchTestCase { + + private PythonScriptEngineService se; + + @Before + public void setup() { + se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); + } + + @After + public void close() { + // We need to clear some system properties + System.clearProperty("python.cachedir.skip"); + System.clearProperty("python.console.encoding"); + se.close(); + } + + @Test + public void testSimpleEquation() { + Map vars = new HashMap(); + Object o = se.execute(se.compile("1 + 2"), vars); + assertThat(((Number) o).intValue(), equalTo(3)); + } + + @Test + public void testMapAccess() { + Map vars = new HashMap(); + + Map obj2 = MapBuilder.newMapBuilder().put("prop2", "value2").map(); + Map obj1 = MapBuilder.newMapBuilder().put("prop1", "value1").put("obj2", obj2).put("l", Arrays.asList("2", "1")).map(); + vars.put("obj1", obj1); + Object o = se.execute(se.compile("obj1"), vars); + assertThat(o, instanceOf(Map.class)); + obj1 = (Map) o; + assertThat((String) obj1.get("prop1"), equalTo("value1")); + assertThat((String) ((Map) obj1.get("obj2")).get("prop2"), equalTo("value2")); + + o = se.execute(se.compile("obj1['l'][0]"), vars); + assertThat(((String) o), equalTo("2")); + } + + @Test + public void testObjectMapInter() { + Map vars = new HashMap(); + Map ctx = new HashMap(); + Map obj1 = new HashMap(); + obj1.put("prop1", "value1"); + ctx.put("obj1", obj1); + vars.put("ctx", ctx); + + se.execute(se.compile("ctx['obj2'] = { 'prop2' : 'value2' }; ctx['obj1']['prop1'] = 'uvalue1'"), vars); + ctx = (Map) se.unwrap(vars.get("ctx")); + assertThat(ctx.containsKey("obj1"), equalTo(true)); + assertThat((String) ((Map) ctx.get("obj1")).get("prop1"), equalTo("uvalue1")); + assertThat(ctx.containsKey("obj2"), equalTo(true)); + assertThat((String) ((Map) ctx.get("obj2")).get("prop2"), equalTo("value2")); + } + + @Test + public void testAccessListInScript() { + + Map vars = new HashMap(); + Map obj2 = MapBuilder.newMapBuilder().put("prop2", "value2").map(); + Map obj1 = MapBuilder.newMapBuilder().put("prop1", "value1").put("obj2", obj2).map(); + vars.put("l", Arrays.asList("1", "2", "3", obj1)); + +// Object o = se.execute(se.compile("l.length"), vars); +// assertThat(((Number) o).intValue(), equalTo(4)); + + Object o = se.execute(se.compile("l[0]"), vars); + assertThat(((String) o), equalTo("1")); + + o = se.execute(se.compile("l[3]"), vars); + obj1 = (Map) o; + assertThat((String) obj1.get("prop1"), equalTo("value1")); + assertThat((String) ((Map) obj1.get("obj2")).get("prop2"), equalTo("value2")); + + o = se.execute(se.compile("l[3]['prop1']"), vars); + assertThat(((String) o), equalTo("value1")); + } + + @Test + public void testChangingVarsCrossExecution1() { + Map vars = new HashMap(); + Map ctx = new HashMap(); + vars.put("ctx", ctx); + Object compiledScript = se.compile("ctx['value']"); + + ExecutableScript script = se.executable(compiledScript, vars); + ctx.put("value", 1); + Object o = script.run(); + assertThat(((Number) o).intValue(), equalTo(1)); + + ctx.put("value", 2); + o = script.run(); + assertThat(((Number) o).intValue(), equalTo(2)); + } + + @Test + public void testChangingVarsCrossExecution2() { + Map vars = new HashMap(); + Map ctx = new HashMap(); + Object compiledScript = se.compile("value"); + + ExecutableScript script = se.executable(compiledScript, vars); + script.setNextVar("value", 1); + Object o = script.run(); + assertThat(((Number) o).intValue(), equalTo(1)); + + script.setNextVar("value", 2); + o = script.run(); + assertThat(((Number) o).intValue(), equalTo(2)); + } +} diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java new file mode 100644 index 00000000000..9d53507388b --- /dev/null +++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.python; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.test.ElasticsearchTestCase; +import org.junit.After; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.hamcrest.Matchers.equalTo; + +/** + * + */ +public class PythonScriptMultiThreadedTest extends ElasticsearchTestCase { + + @After + public void close() { + // We need to clear some system properties + System.clearProperty("python.cachedir.skip"); + System.clearProperty("python.console.encoding"); + } + + @Test + public void testExecutableNoRuntimeParams() throws Exception { + final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); + final Object compiled = se.compile("x + y"); + final AtomicBoolean failed = new AtomicBoolean(); + + Thread[] threads = new Thread[4]; + final CountDownLatch latch = new CountDownLatch(threads.length); + final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1); + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(new Runnable() { + @Override + public void run() { + try { + barrier.await(); + long x = ThreadLocalRandom.current().nextInt(); + long y = ThreadLocalRandom.current().nextInt(); + long addition = x + y; + Map vars = new HashMap(); + vars.put("x", x); + vars.put("y", y); + ExecutableScript script = se.executable(compiled, vars); + for (int i = 0; i < 10000; i++) { + long result = ((Number) script.run()).longValue(); + assertThat(result, equalTo(addition)); + } + } catch (Throwable t) { + failed.set(true); + logger.error("failed", t); + } finally { + latch.countDown(); + } + } + }); + } + for (int i = 0; i < threads.length; i++) { + threads[i].start(); + } + barrier.await(); + latch.await(); + assertThat(failed.get(), equalTo(false)); + } + + +// @Test public void testExecutableWithRuntimeParams() throws Exception { +// final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); +// final Object compiled = se.compile("x + y"); +// final AtomicBoolean failed = new AtomicBoolean(); +// +// Thread[] threads = new Thread[50]; +// final CountDownLatch latch = new CountDownLatch(threads.length); +// final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1); +// for (int i = 0; i < threads.length; i++) { +// threads[i] = new Thread(new Runnable() { +// @Override public void run() { +// try { +// barrier.await(); +// long x = ThreadLocalRandom.current().nextInt(); +// Map vars = new HashMap(); +// vars.put("x", x); +// ExecutableScript script = se.executable(compiled, vars); +// Map runtimeVars = new HashMap(); +// for (int i = 0; i < 100000; i++) { +// long y = ThreadLocalRandom.current().nextInt(); +// long addition = x + y; +// runtimeVars.put("y", y); +// long result = ((Number) script.run(runtimeVars)).longValue(); +// assertThat(result, equalTo(addition)); +// } +// } catch (Throwable t) { +// failed.set(true); +// logger.error("failed", t); +// } finally { +// latch.countDown(); +// } +// } +// }); +// } +// for (int i = 0; i < threads.length; i++) { +// threads[i].start(); +// } +// barrier.await(); +// latch.await(); +// assertThat(failed.get(), equalTo(false)); +// } + + @Test + public void testExecute() throws Exception { + final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); + final Object compiled = se.compile("x + y"); + final AtomicBoolean failed = new AtomicBoolean(); + + Thread[] threads = new Thread[4]; + final CountDownLatch latch = new CountDownLatch(threads.length); + final CyclicBarrier barrier = new CyclicBarrier(threads.length + 1); + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(new Runnable() { + @Override + public void run() { + try { + barrier.await(); + Map runtimeVars = new HashMap(); + for (int i = 0; i < 10000; i++) { + long x = ThreadLocalRandom.current().nextInt(); + long y = ThreadLocalRandom.current().nextInt(); + long addition = x + y; + runtimeVars.put("x", x); + runtimeVars.put("y", y); + long result = ((Number) se.execute(compiled, runtimeVars)).longValue(); + assertThat(result, equalTo(addition)); + } + } catch (Throwable t) { + failed.set(true); + logger.error("failed", t); + } finally { + latch.countDown(); + } + } + }); + } + for (int i = 0; i < threads.length; i++) { + threads[i].start(); + } + barrier.await(); + latch.await(); + assertThat(failed.get(), equalTo(false)); + } +} diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java new file mode 100644 index 00000000000..0ff68beea3e --- /dev/null +++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java @@ -0,0 +1,311 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.python; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.hamcrest.CoreMatchers; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.client.Requests.searchRequest; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.QueryBuilders.*; +import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; +import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.equalTo; + +/** + * + */ +@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.SUITE) +public class PythonScriptSearchTests extends ElasticsearchIntegrationTest { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .build(); + } + + @After + public void close() { + // We need to clear some system properties + System.clearProperty("python.cachedir.skip"); + System.clearProperty("python.console.encoding"); + } + + @Test + public void testPythonFilter() throws Exception { + createIndex("test"); + index("test", "type1", "1", jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject()); + flush(); + index("test", "type1", "2", jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject()); + flush(); + index("test", "type1", "3", jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).endObject()); + refresh(); + + logger.info(" --> running doc['num1'].value > 1"); + SearchResponse response = client().prepareSearch() + .setQuery(filteredQuery(matchAllQuery(), scriptQuery("doc['num1'].value > 1").lang("python"))) + .addSort("num1", SortOrder.ASC) + .addScriptField("sNum1", "python", "doc['num1'].value", null) + .execute().actionGet(); + + assertThat(response.getHits().totalHits(), equalTo(2l)); + assertThat(response.getHits().getAt(0).id(), equalTo("2")); + assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0)); + assertThat(response.getHits().getAt(1).id(), equalTo("3")); + assertThat((Double) response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(3.0)); + + logger.info(" --> running doc['num1'].value > param1"); + response = client().prepareSearch() + .setQuery(filteredQuery(matchAllQuery(), scriptQuery("doc['num1'].value > param1").lang("python").addParam("param1", 2))) + .addSort("num1", SortOrder.ASC) + .addScriptField("sNum1", "python", "doc['num1'].value", null) + .execute().actionGet(); + + assertThat(response.getHits().totalHits(), equalTo(1l)); + assertThat(response.getHits().getAt(0).id(), equalTo("3")); + assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(3.0)); + + logger.info(" --> running doc['num1'].value > param1"); + response = client().prepareSearch() + .setQuery(filteredQuery(matchAllQuery(), scriptQuery("doc['num1'].value > param1").lang("python").addParam("param1", -1))) + .addSort("num1", SortOrder.ASC) + .addScriptField("sNum1", "python", "doc['num1'].value", null) + .execute().actionGet(); + + assertThat(response.getHits().totalHits(), equalTo(3l)); + assertThat(response.getHits().getAt(0).id(), equalTo("1")); + assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0)); + assertThat(response.getHits().getAt(1).id(), equalTo("2")); + assertThat((Double) response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0)); + assertThat(response.getHits().getAt(2).id(), equalTo("3")); + assertThat((Double) response.getHits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0)); + } + + @Test + public void testScriptFieldUsingSource() throws Exception { + createIndex("test"); + index("test", "type1", "1", + jsonBuilder().startObject() + .startObject("obj1").field("test", "something").endObject() + .startObject("obj2").startArray("arr2").value("arr_value1").value("arr_value2").endArray().endObject() + .endObject()); + refresh(); + + SearchResponse response = client().prepareSearch() + .setQuery(matchAllQuery()) + .addScriptField("s_obj1", "python", "_source['obj1']", null) + .addScriptField("s_obj1_test", "python", "_source['obj1']['test']", null) + .addScriptField("s_obj2", "python", "_source['obj2']", null) + .addScriptField("s_obj2_arr2", "python", "_source['obj2']['arr2']", null) + .execute().actionGet(); + + Map sObj1 = (Map) response.getHits().getAt(0).field("s_obj1").value(); + assertThat(sObj1.get("test").toString(), equalTo("something")); + assertThat(response.getHits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something")); + + Map sObj2 = (Map) response.getHits().getAt(0).field("s_obj2").value(); + List sObj2Arr2 = (List) sObj2.get("arr2"); + assertThat(sObj2Arr2.size(), equalTo(2)); + assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1")); + assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2")); + + sObj2Arr2 = (List) response.getHits().getAt(0).field("s_obj2_arr2").values(); + assertThat(sObj2Arr2.size(), equalTo(2)); + assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1")); + assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2")); + } + + @Test + public void testCustomScriptBoost() throws Exception { + createIndex("test"); + index("test", "type1", "1", jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject()); + index("test", "type1", "2", jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject()); + refresh(); + + logger.info("--- QUERY_THEN_FETCH"); + + logger.info(" --> running doc['num1'].value"); + SearchResponse response = client().search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("doc['num1'].value").lang("python")))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); + + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + assertThat(response.getHits().getAt(0).id(), equalTo("2")); + assertThat(response.getHits().getAt(1).id(), equalTo("1")); + + logger.info(" --> running -doc['num1'].value"); + response = client().search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("-doc['num1'].value").lang("python")))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); + + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + assertThat(response.getHits().getAt(0).id(), equalTo("1")); + assertThat(response.getHits().getAt(1).id(), equalTo("2")); + + + logger.info(" --> running doc['num1'].value * _score"); + response = client().search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("doc['num1'].value * _score.doubleValue()").lang("python")))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); + + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + assertThat(response.getHits().getAt(0).id(), equalTo("2")); + assertThat(response.getHits().getAt(1).id(), equalTo("1")); + + logger.info(" --> running param1 * param2 * _score"); + response = client().search(searchRequest() + .searchType(SearchType.QUERY_THEN_FETCH) + .source(searchSource().explain(true).query(functionScoreQuery(termQuery("test", "value")) + .add(ScoreFunctionBuilders.scriptFunction("param1 * param2 * _score.doubleValue()").param("param1", 2).param("param2", 2).lang("python")))) + ).actionGet(); + + assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0)); + + assertThat(response.getHits().totalHits(), equalTo(2l)); + logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation()); + logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation()); + } + + /** + * Test case for #4: https://github.com/elasticsearch/elasticsearch-lang-python/issues/4 + * Update request that uses python script with no parameters fails with NullPointerException + * @throws Exception + */ + @Test + public void testPythonEmptyParameters() throws Exception { + createIndex("test"); + index("test", "type1", "1", jsonBuilder().startObject().field("myfield", "foo").endObject()); + refresh(); + + client().prepareUpdate("test", "type1", "1").setScriptLang("python") + .setScript("ctx[\"_source\"][\"myfield\"]=\"bar\"", ScriptService.ScriptType.INLINE) + .execute().actionGet(); + refresh(); + + Object value = get("test", "type1", "1").getSourceAsMap().get("myfield"); + assertThat(value instanceof String, is(true)); + + assertThat((String) value, CoreMatchers.equalTo("bar")); + } + + @Test + public void testScriptScoresNested() throws IOException { + createIndex("index"); + ensureYellow(); + index("index", "testtype", "1", jsonBuilder().startObject().field("dummy_field", 1).endObject()); + refresh(); + SearchResponse response = client().search( + searchRequest().source( + searchSource().query( + functionScoreQuery( + functionScoreQuery( + functionScoreQuery().add(scriptFunction("1").lang("python"))) + .add(scriptFunction("_score.doubleValue()").lang("python"))) + .add(scriptFunction("_score.doubleValue()").lang("python") + ) + ) + ) + ).actionGet(); + assertSearchResponse(response); + assertThat(response.getHits().getAt(0).score(), equalTo(1.0f)); + } + + @Test + public void testScriptScoresWithAgg() throws IOException { + createIndex("index"); + ensureYellow(); + index("index", "testtype", "1", jsonBuilder().startObject().field("dummy_field", 1).endObject()); + refresh(); + SearchResponse response = client().search( + searchRequest().source( + searchSource().query( + functionScoreQuery() + .add(scriptFunction("_score.doubleValue()").lang("python") + ) + ).aggregation(terms("score_agg").script("_score.doubleValue()").lang("python")) + ) + ).actionGet(); + assertSearchResponse(response); + assertThat(response.getHits().getAt(0).score(), equalTo(1.0f)); + assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getKeyAsNumber().floatValue(), Matchers.is(1f)); + assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getDocCount(), Matchers.is(1l)); + } + + /** + * Test case for #19: https://github.com/elasticsearch/elasticsearch-lang-python/issues/19 + * Multi-line or multi-statement Python scripts raise NullPointerException + */ + @Test + public void testPythonMultiLines() throws Exception { + createIndex("test"); + index("test", "type1", "1", jsonBuilder().startObject().field("myfield", "foo").endObject()); + refresh(); + + client().prepareUpdate("test", "type1", "1").setScriptLang("python") + .setScript("a=42; ctx[\"_source\"][\"myfield\"]=\"bar\"", ScriptService.ScriptType.INLINE) + .execute().actionGet(); + refresh(); + + Object value = get("test", "type1", "1").getSourceAsMap().get("myfield"); + assertThat(value instanceof String, is(true)); + + assertThat((String) value, CoreMatchers.equalTo("bar")); + } + +} diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/SimpleBench.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/SimpleBench.java new file mode 100644 index 00000000000..583bab163fa --- /dev/null +++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/SimpleBench.java @@ -0,0 +1,71 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.python; + +import org.elasticsearch.common.StopWatch; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ExecutableScript; + +import java.util.HashMap; +import java.util.Map; + +/** + * + */ +public class SimpleBench { + + public static void main(String[] args) { + PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS); + Object compiled = se.compile("x + y"); + + Map vars = new HashMap(); + // warm up + for (int i = 0; i < 1000; i++) { + vars.put("x", i); + vars.put("y", i + 1); + se.execute(compiled, vars); + } + + final long ITER = 100000; + + StopWatch stopWatch = new StopWatch().start(); + for (long i = 0; i < ITER; i++) { + se.execute(compiled, vars); + } + System.out.println("Execute Took: " + stopWatch.stop().lastTaskTime()); + + stopWatch = new StopWatch().start(); + ExecutableScript executableScript = se.executable(compiled, vars); + for (long i = 0; i < ITER; i++) { + executableScript.run(); + } + System.out.println("Executable Took: " + stopWatch.stop().lastTaskTime()); + + stopWatch = new StopWatch().start(); + executableScript = se.executable(compiled, vars); + for (long i = 0; i < ITER; i++) { + for (Map.Entry entry : vars.entrySet()) { + executableScript.setNextVar(entry.getKey(), entry.getValue()); + } + executableScript.run(); + } + System.out.println("Executable (vars) Took: " + stopWatch.stop().lastTaskTime()); + } +} diff --git a/plugins/pom.xml b/plugins/pom.xml index ca5b8ac5cce..7e54c2207be 100644 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -128,5 +128,7 @@ cloud-gce cloud-azure cloud-aws + lang-python + lang-javascript