Add Groovy as a scripting language, add sandboxing for Groovy

Sandboxes the groovy scripting language with multiple configurable
whitelists:

`script.groovy.sandbox.receiver_whitelist`: comma-separated list of string
classes for objects that may have methods invoked.
`script.groovy.sandbox.package_whitelist`: comma-separated list of
packages under which new objects may be constructed.
`script.groovy.sandbox.class_whitelist` comma-separated list of classes
that are allowed to be constructed.

As well as a method blacklist:

`script.groovy.sandbox.method_blacklist`: comma-separated list of
methods that are never allowed to be invoked, regardless of target
object.

The sandbox can be entirely disabled by setting:

`script.groovy.sandbox.enabled: false`
This commit is contained in:
Lee Hinman 2014-05-06 10:58:41 +02:00
parent 12fd6ce98c
commit c70f6d0171
21 changed files with 738 additions and 59 deletions

View File

@ -27,10 +27,11 @@ grant {
permission java.io.FilePermission "${junit4.childvm.cwd}", "read,execute,write";
permission java.io.FilePermission "${junit4.childvm.cwd}${/}-", "read,execute,write,delete";
permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,execute,write,delete";
permission groovy.security.GroovyCodeSourcePermission "/groovy/script";
// Allow connecting to the internet anywhere
permission java.net.SocketPermission "*", "accept,listen,connect,resolve";
// Basic permissions needed for Lucene / Elasticsearch to work:
permission java.util.PropertyPermission "*", "read,write";
permission java.lang.reflect.ReflectPermission "*";

36
pom.xml
View File

@ -208,13 +208,6 @@
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.mvel</groupId>
<artifactId>mvel2</artifactId>
<version>2.2.0.Final</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
@ -265,6 +258,22 @@
</dependency>
<!-- END: dependencies that are shaded -->
<dependency>
<groupId>org.mvel</groupId>
<artifactId>mvel2</artifactId>
<version>2.2.0.Final</version>
<scope>compile</scope>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-all</artifactId>
<version>2.3.2</version>
<scope>compile</scope>
<optional>true</optional>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
@ -648,7 +657,6 @@
<includes>
<include>com.google.guava:guava</include>
<include>com.carrotsearch:hppc</include>
<include>org.mvel:mvel2</include>
<include>com.fasterxml.jackson.core:jackson-core</include>
<include>com.fasterxml.jackson.dataformat:jackson-dataformat-smile</include>
<include>com.fasterxml.jackson.dataformat:jackson-dataformat-yaml</include>
@ -674,10 +682,6 @@
<pattern>jsr166e</pattern>
<shadedPattern>org.elasticsearch.common.util.concurrent.jsr166e</shadedPattern>
</relocation>
<relocation>
<pattern>org.mvel2</pattern>
<shadedPattern>org.elasticsearch.common.mvel2</shadedPattern>
</relocation>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
<shadedPattern>org.elasticsearch.common.jackson</shadedPattern>
@ -870,7 +874,7 @@
</data>
<data>
<src>${project.build.directory}/lib</src>
<includes>lucene*, log4j*, jna*, spatial4j*, jts*</includes>
<includes>lucene*, log4j*, jna*, spatial4j*, jts*, groovy*, mvel*</includes>
<type>directory</type>
<mapper>
<type>perm</type>
@ -1070,6 +1074,8 @@
<include>jna*</include>
<include>spatial4j*</include>
<include>jts*</include>
<include>groovy*</include>
<include>mvel*</include>
</includes>
</source>
<source>
@ -1393,7 +1399,7 @@
<version>0.6.4.201312101107</version>
<scope>test</scope>
</dependency>
</dependencies>
</dependencies>
<build>
<plugins>
<plugin>
@ -1418,7 +1424,7 @@
<id>default-check</id>
<goals>
<goal>check</goal>
</goals>
</goals>
</execution>
</executions>
<configuration>

View File

@ -9,6 +9,8 @@
<include>net.java.dev.jna:jna</include>
<include>com.spatial4j:spatial4j</include>
<include>com.vividsolutions:jts</include>
<include>org.codehaus.groovy:groovy-all</include>
<include>org.mvel:mvel2</include>
</includes>
</dependencySet>
<dependencySet>

View File

@ -27,6 +27,7 @@ import org.elasticsearch.common.inject.multibindings.MapBinder;
import org.elasticsearch.common.inject.multibindings.Multibinder;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.script.groovy.GroovyScriptEngineService;
import org.elasticsearch.script.mustache.MustacheScriptEngineService;
import org.elasticsearch.script.mvel.MvelScriptEngineService;
@ -77,16 +78,23 @@ public class ScriptModule extends AbstractModule {
Multibinder<ScriptEngineService> multibinder = Multibinder.newSetBinder(binder(), ScriptEngineService.class);
multibinder.addBinding().to(NativeScriptEngineService.class);
try {
multibinder.addBinding().to(GroovyScriptEngineService.class);
} catch (Throwable t) {
Loggers.getLogger(GroovyScriptEngineService.class).debug("failed to load groovy", t);
}
try {
multibinder.addBinding().to(MvelScriptEngineService.class);
} catch (Throwable t) {
// no MVEL
Loggers.getLogger(MvelScriptEngineService.class).debug("failed to load mvel", t);
}
try {
multibinder.addBinding().to(MustacheScriptEngineService.class);
} catch (Throwable t) {
Loggers.getLogger(MustacheScriptEngineService.class).trace("failed to load mustache", t);
Loggers.getLogger(MustacheScriptEngineService.class).debug("failed to load mustache", t);
}
for (Class<? extends ScriptEngineService> scriptEngine : scriptEngines) {

View File

@ -44,6 +44,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
@ -54,6 +55,10 @@ import java.util.concurrent.TimeUnit;
*/
public class ScriptService extends AbstractComponent {
public static final String DEFAULT_SCRIPTING_LANGUAGE_SETTING = "script.default_lang";
public static final String DISABLE_DYNAMIC_SCRIPTING_SETTING = "script.disable_dynamic";
public static final String DISABLE_DYNAMIC_SCRIPTING_DEFAULT = "sandbox";
private final String defaultLang;
private final ImmutableMap<String, ScriptEngineService> scriptEngines;
@ -63,7 +68,38 @@ public class ScriptService extends AbstractComponent {
private final Cache<CacheKey, CompiledScript> cache;
private final File scriptsDirectory;
private final boolean disableDynamic;
private final DynamicScriptDisabling dynamicScriptingDisabled;
/**
* Enum defining the different dynamic settings for scripting, either
* ONLY_DISK_ALLOWED (scripts must be placed on disk), EVERYTHING_ALLOWED
* (all dynamic scripting is enabled), or SANDBOXED_ONLY (only sandboxed
* scripting languages are allowed)
*/
enum DynamicScriptDisabling {
EVERYTHING_ALLOWED,
ONLY_DISK_ALLOWED,
SANDBOXED_ONLY;
public static final DynamicScriptDisabling parse(String s) {
switch (s.toLowerCase(Locale.ROOT)) {
// true for "disable_dynamic" means only on-disk scripts are enabled
case "true":
case "all":
return ONLY_DISK_ALLOWED;
// false for "disable_dynamic" means all scripts are enabled
case "false":
case "none":
return EVERYTHING_ALLOWED;
// only sandboxed scripting is enabled
case "sandbox":
case "sandboxed":
return SANDBOXED_ONLY;
default:
throw new ElasticsearchIllegalArgumentException("Unrecognized script allowance setting: [" + s + "]");
}
}
}
@Inject
public ScriptService(Settings settings, Environment env, Set<ScriptEngineService> scriptEngines,
@ -74,8 +110,8 @@ public class ScriptService extends AbstractComponent {
TimeValue cacheExpire = componentSettings.getAsTime("cache.expire", null);
logger.debug("using script cache with max_size [{}], expire [{}]", cacheMaxSize, cacheExpire);
this.defaultLang = componentSettings.get("default_lang", "mvel");
this.disableDynamic = componentSettings.getAsBoolean("disable_dynamic", true);
this.defaultLang = settings.get(DEFAULT_SCRIPTING_LANGUAGE_SETTING, "mvel");
this.dynamicScriptingDisabled = DynamicScriptDisabling.parse(settings.get(DISABLE_DYNAMIC_SCRIPTING_SETTING, DISABLE_DYNAMIC_SCRIPTING_DEFAULT));
CacheBuilder cacheBuilder = CacheBuilder.newBuilder();
if (cacheMaxSize >= 0) {
@ -130,7 +166,7 @@ public class ScriptService extends AbstractComponent {
lang = defaultLang;
}
if (!dynamicScriptEnabled(lang)) {
throw new ScriptException("dynamic scripting disabled");
throw new ScriptException("dynamic scripting for [" + lang + "] disabled");
}
CacheKey cacheKey = new CacheKey(lang, script);
compiled = cache.getIfPresent(cacheKey);
@ -180,12 +216,16 @@ public class ScriptService extends AbstractComponent {
if (service == null) {
throw new ElasticsearchIllegalArgumentException("script_lang not supported [" + lang + "]");
}
// Templating languages and native scripts are always allowed
// "native" executions are registered through plugins
if (service.sandboxed() || "native".equals(lang)) {
// Templating languages (mustache) and native scripts are always
// allowed, "native" executions are registered through plugins
if (this.dynamicScriptingDisabled == DynamicScriptDisabling.EVERYTHING_ALLOWED || "native".equals(lang) || "mustache".equals(lang)) {
return true;
} else if (this.dynamicScriptingDisabled == DynamicScriptDisabling.ONLY_DISK_ALLOWED) {
return false;
} else {
return service.sandboxed();
}
return !disableDynamic;
}
private class ScriptChangesListener extends FileChangesListener {

View File

@ -0,0 +1,157 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.groovy;
import com.google.common.collect.ImmutableSet;
import org.codehaus.groovy.ast.ClassNode;
import org.codehaus.groovy.ast.expr.ConstructorCallExpression;
import org.codehaus.groovy.ast.expr.Expression;
import org.codehaus.groovy.ast.expr.MethodCallExpression;
import org.codehaus.groovy.control.customizers.SecureASTCustomizer;
import org.elasticsearch.common.settings.Settings;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import static com.google.common.collect.Lists.newArrayList;
/**
* Class used to determine whether a Groovy expression should be allowed.
* During compilation, every expression is passed to the
* <code>isAuthorized</code> method, which returns true to allow that method
* and false to block it. Includes all of the sandbox-related whitelist and
* blacklist options.
*/
public class GroovySandboxExpressionChecker implements SecureASTCustomizer.ExpressionChecker {
public static String GROOVY_SANDBOX_METHOD_BLACKLIST = "script.groovy.sandbox.method_blacklist";
public static String GROOVY_SANDBOX_PACKAGE_WHITELIST = "script.groovy.sandbox.package_whitelist";
public static String GROOVY_SANDBOX_CLASS_WHITELIST = "script.groovy.sandbox.class_whitelist";
public static String GROOVY_SCRIPT_SANDBOX_RECEIVER_WHITELIST = "script.groovy.sandbox.receiver_whitelist";
private final Set<String> methodBlacklist;
private final Set<String> packageWhitelist;
private final Set<String> classWhitelist;
public GroovySandboxExpressionChecker(Settings settings) {
this.methodBlacklist = ImmutableSet.copyOf(settings.getAsArray(GROOVY_SANDBOX_METHOD_BLACKLIST, defaultMethodBlacklist, true));
this.packageWhitelist = ImmutableSet.copyOf(settings.getAsArray(GROOVY_SANDBOX_PACKAGE_WHITELIST, defaultPackageWhitelist, true));
this.classWhitelist = ImmutableSet.copyOf(settings.getAsArray(GROOVY_SANDBOX_CLASS_WHITELIST, defaultClassConstructionWhitelist, true));
}
// Never allow calling these methods, regardless of the object type
public static String[] defaultMethodBlacklist = new String[]{
"getClass",
"wait",
"notify",
"notifyAll",
"finalize"
};
// Only instances of these classes in these packages can be instantiated
public static String[] defaultPackageWhitelist = new String[] {"java.util", "java.lang", "org.joda.time"};
// Classes that are allowed to be constructed
public static String[] defaultClassConstructionWhitelist = new String[]{
java.util.Date.class.getName(),
java.util.Map.class.getName(),
java.util.List.class.getName(),
java.util.Set.class.getName(),
java.util.ArrayList.class.getName(),
java.util.Arrays.class.getName(),
java.util.HashMap.class.getName(),
java.util.HashSet.class.getName(),
java.util.UUID.class.getName(),
org.joda.time.DateTime.class.getName(),
org.joda.time.DateTimeZone.class.getName()
};
// Default whitelisted receiver classes for the Groovy sandbox
private final static String[] defaultReceiverWhitelist = new String [] {
java.lang.Math.class.getName(),
java.lang.Integer.class.getName(), "[I", "[[I", "[[[I",
java.lang.Float.class.getName(), "[F", "[[F", "[[[F",
java.lang.Double.class.getName(), "[D", "[[D", "[[[D",
java.lang.Long.class.getName(), "[J", "[[J", "[[[J",
java.lang.Short.class.getName(), "[S", "[[S", "[[[S",
java.lang.Character.class.getName(), "[C", "[[C", "[[[C",
java.lang.Byte.class.getName(), "[B", "[[B", "[[[B",
java.lang.Boolean.class.getName(), "[Z", "[[Z", "[[[Z",
java.math.BigDecimal.class.getName(),
java.util.Arrays.class.getName(),
java.util.Date.class.getName(),
java.util.List.class.getName(),
java.util.Map.class.getName(),
java.util.Set.class.getName(),
java.lang.Object.class.getName(),
org.joda.time.DateTime.class.getName(),
org.joda.time.DateTimeUtils.class.getName(),
org.joda.time.DateTimeZone.class.getName(),
org.joda.time.Instant.class.getName()
};
/**
* Checks whether the expression to be compiled is allowed
*/
@Override
public boolean isAuthorized(Expression expression) {
if (expression instanceof MethodCallExpression) {
MethodCallExpression mce = (MethodCallExpression) expression;
if (methodBlacklist.contains(mce.getMethodAsString())) {
return false;
}
} else if (expression instanceof ConstructorCallExpression) {
ConstructorCallExpression cce = (ConstructorCallExpression) expression;
ClassNode type = cce.getType();
if (!packageWhitelist.contains(type.getPackageName())) {
return false;
}
if (!classWhitelist.contains(type.getName())) {
return false;
}
}
return true;
}
/**
* Returns a customized ASTCustomizer that includes the whitelists and
* expression checker.
*/
public static SecureASTCustomizer getSecureASTCustomizer(Settings settings) {
SecureASTCustomizer scz = new SecureASTCustomizer();
// Closures are allowed
scz.setClosuresAllowed(true);
// But defining methods is not
scz.setMethodDefinitionAllowed(false);
// Only allow the imports that we explicitly call out
List<String> importWhitelist = new ArrayList<>();
importWhitelist.addAll(ImmutableSet.copyOf(GroovySandboxExpressionChecker.defaultClassConstructionWhitelist));
scz.setImportsWhitelist(importWhitelist);
// Package definitions are not allowed
scz.setPackageAllowed(false);
// White-listed receivers of method calls
String[] receiverWhitelist = settings.getAsArray(GROOVY_SCRIPT_SANDBOX_RECEIVER_WHITELIST, defaultReceiverWhitelist, true);
scz.setReceiversWhiteList(newArrayList(receiverWhitelist));
// Add the customized expression checker for finer-grained checking
scz.addExpressionCheckers(new GroovySandboxExpressionChecker(settings));
return scz;
}
}

View File

@ -0,0 +1,287 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.groovy;
import groovy.lang.Binding;
import groovy.lang.GroovyClassLoader;
import groovy.lang.Script;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.Scorer;
import org.codehaus.groovy.control.CompilerConfiguration;
import org.codehaus.groovy.control.customizers.ImportCustomizer;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.script.ScriptEngineService;
import org.elasticsearch.script.ScriptException;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.lookup.SearchLookup;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
/**
* Provides the infrastructure for Groovy as a scripting language for Elasticsearch
*/
public class GroovyScriptEngineService extends AbstractComponent implements ScriptEngineService {
public static String GROOVY_SCRIPT_SANDBOX_ENABLED = "script.groovy.sandbox.enabled";
private final AtomicLong counter = new AtomicLong();
private final GroovyClassLoader loader;
private final boolean sandboxed;
@Inject
public GroovyScriptEngineService(Settings settings) {
super(settings);
ImportCustomizer imports = new ImportCustomizer();
imports.addStarImports("org.joda.time");
imports.addStaticStars("java.lang.Math");
CompilerConfiguration config = new CompilerConfiguration();
config.addCompilationCustomizers(imports);
this.sandboxed = settings.getAsBoolean(GROOVY_SCRIPT_SANDBOX_ENABLED, true);
if (this.sandboxed) {
config.addCompilationCustomizers(GroovySandboxExpressionChecker.getSecureASTCustomizer(settings));
}
this.loader = new GroovyClassLoader(settings.getClassLoader(), config);
}
@Override
public void close() {
loader.clearCache();
try {
loader.close();
} catch (IOException e) {
logger.warn("Unable to close Groovy loader", e);
}
}
@Override
public String[] types() {
return new String[]{"groovy"};
}
@Override
public String[] extensions() {
return new String[]{"groovy"};
}
@Override
public boolean sandboxed() {
return this.sandboxed;
}
@Override
public Object compile(String script) {
return loader.parseClass(script, generateScriptName());
}
/**
* Return a script object with the given vars from the compiled script object
*/
private Script createScript(Object compiledScript, Map<String, Object> vars) throws InstantiationException, IllegalAccessException {
Class scriptClass = (Class) compiledScript;
Script scriptObject = (Script) scriptClass.newInstance();
Binding binding = new Binding();
binding.getVariables().putAll(vars);
scriptObject.setBinding(binding);
return scriptObject;
}
@SuppressWarnings({"unchecked"})
@Override
public ExecutableScript executable(Object compiledScript, Map<String, Object> vars) {
try {
Map<String, Object> allVars = new HashMap<>();
if (vars != null) {
allVars.putAll(vars);
}
return new GroovyScript(createScript(compiledScript, allVars));
} catch (Exception e) {
throw new ScriptException("failed to build executable script", e);
}
}
@SuppressWarnings({"unchecked"})
@Override
public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable Map<String, Object> vars) {
try {
Map<String, Object> allVars = new HashMap<>();
allVars.putAll(lookup.asMap());
if (vars != null) {
allVars.putAll(vars);
}
Script scriptObject = createScript(compiledScript, allVars);
return new GroovyScript(scriptObject, lookup);
} catch (Exception e) {
throw new ScriptException("failed to build search script", e);
}
}
@Override
public Object execute(Object compiledScript, Map<String, Object> vars) {
try {
Map<String, Object> allVars = new HashMap<>();
if (vars != null) {
allVars.putAll(vars);
}
Script scriptObject = createScript(compiledScript, allVars);
return scriptObject.run();
} catch (Exception e) {
throw new ScriptException("failed to execute script", e);
}
}
@Override
public Object unwrap(Object value) {
return value;
}
private String generateScriptName() {
return "Script" + counter.incrementAndGet() + ".groovy";
}
public static final class GroovyScript implements ExecutableScript, SearchScript {
private final Script script;
private final SearchLookup lookup;
private final Map<String, Object> variables;
private final UpdateableFloat score;
public GroovyScript(Script script) {
this(script, null);
}
public GroovyScript(Script script, SearchLookup lookup) {
this.script = script;
this.lookup = lookup;
this.variables = script.getBinding().getVariables();
this.score = new UpdateableFloat(0);
// Add the _score variable, which will be updated per-document by
// setting .value on the UpdateableFloat instance
this.variables.put("_score", this.score);
}
@Override
public void setScorer(Scorer scorer) {
if (lookup != null) {
lookup.setScorer(scorer);
}
}
@Override
public void setNextReader(AtomicReaderContext context) {
if (lookup != null) {
lookup.setNextReader(context);
}
}
@Override
public void setNextDocId(int doc) {
if (lookup != null) {
lookup.setNextDocId(doc);
}
}
@SuppressWarnings({"unchecked"})
@Override
public void setNextScore(float score) {
this.score.value = score;
}
@SuppressWarnings({"unchecked"})
@Override
public void setNextVar(String name, Object value) {
variables.put(name, value);
}
@Override
public void setNextSource(Map<String, Object> source) {
if (lookup != null) {
lookup.source().setNextSource(source);
}
}
@Override
public Object run() {
return script.run();
}
@Override
public float runAsFloat() {
return ((Number) run()).floatValue();
}
@Override
public long runAsLong() {
return ((Number) run()).longValue();
}
@Override
public double runAsDouble() {
return ((Number) run()).doubleValue();
}
@Override
public Object unwrap(Object value) {
return value;
}
/**
* Float encapsulation that allows updating the value with public
* member access. This is used to encapsulate the _score of a document
* so that updating the _score for the next document incurs only the
* overhead of setting a member variable
*/
private final class UpdateableFloat extends Number {
public float value;
public UpdateableFloat(float value) {
this.value = value;
}
@Override
public int intValue() {
return (int)value;
}
@Override
public long longValue() {
return (long)value;
}
@Override
public float floatValue() {
return value;
}
@Override
public double doubleValue() {
return value;
}
}
}
}

View File

@ -28,6 +28,7 @@ import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.functionscore.script.ScriptScoreFunctionBuilder;
import org.elasticsearch.script.groovy.GroovyScriptEngineService;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.junit.After;
import org.junit.Before;
@ -66,14 +67,14 @@ public class BenchmarkIntegrationTest extends ElasticsearchIntegrationTest {
protected synchronized Settings nodeSettings(int nodeOrdinal) {
if (nodeOrdinal == 0) { // at least one
return ImmutableSettings.builder().put("node.bench", true).build();
return ImmutableSettings.builder().put("node.bench", true).put(GroovyScriptEngineService.GROOVY_SCRIPT_SANDBOX_ENABLED, false).build();
} else {
if (benchNodes.containsKey(nodeOrdinal)) {
return ImmutableSettings.builder().put("node.bench", benchNodes.get(nodeOrdinal)).build();
return ImmutableSettings.builder().put("node.bench", benchNodes.get(nodeOrdinal)).put(GroovyScriptEngineService.GROOVY_SCRIPT_SANDBOX_ENABLED, false).build();
} else {
boolean b = randomBoolean();
benchNodes.put(nodeOrdinal, b);
return ImmutableSettings.builder().put("node.bench", b).build();
return ImmutableSettings.builder().put("node.bench", b).put(GroovyScriptEngineService.GROOVY_SCRIPT_SANDBOX_ENABLED, false).build();
}
}

View File

@ -0,0 +1,107 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.search.SearchPhaseExecutionException;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.junit.Test;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
import static org.hamcrest.CoreMatchers.equalTo;
/**
* Tests for the Groovy scripting sandbox
*/
public class GroovySandboxScriptTests extends ElasticsearchIntegrationTest {
@Test
public void testSandboxedGroovyScript() {
client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefresh(true).get();
// Plain test
testSuccess("");
// List
testSuccess("def list = [doc['foo'].value, 3, 4]; def v = list.get(1); list.add(10)");
// Ranges
testSuccess("def range = 1..doc['foo'].value; def v = range.get(0)");
// Maps
testSuccess("def v = doc['foo'].value; def m = [:]; m.put(\\\"value\\\", v)");
// Times
testSuccess("def t = Instant.now().getMillis()");
// Fail cases
testFailure("pr = Runtime.getRuntime().exec(\\\"touch /tmp/gotcha\\\"); pr.waitFor()",
"Method calls not allowed on [java.lang.Runtime]");
testFailure("d = new DateTime(); d.getClass().getDeclaredMethod(\\\"plus\\\").setAccessible(true)",
"Expression [MethodCallExpression] is not allowed: d.getClass()");
testFailure("Class.forName(\\\"DateTime\\\").getDeclaredMethod(\\\"plus\\\").setAccessible(true)",
"Method calls not allowed on [java.lang.Class]");
testFailure("Eval.me('2 + 2')", "Method calls not allowed on [groovy.util.Eval]");
testFailure("Eval.x(5, 'x + 2')", "Method calls not allowed on [groovy.util.Eval]");
testFailure("t = new java.util.concurrent.ThreadPoolExecutor(2, 2, 0L, TimeUnit.MILLISECONDS, " +
"new java.util.concurrent.LinkedBlockingQueue<Runnable>()); t.execute({ println 5 })",
"Expression [ConstructorCallExpression] is not allowed: new java.util.concurrent.ThreadPoolExecutor");
testFailure("d = new Date(); java.lang.reflect.Field f = Date.class.getDeclaredField(\\\"fastTime\\\");" +
" f.setAccessible(true); f.get(\\\"fastTime\\\")",
"Method calls not allowed on [java.lang.reflect.Field]");
testFailure("t = new Thread({ println 3 }); t.start(); t.join()",
"Expression [ConstructorCallExpression] is not allowed: new java.lang.Thread");
testFailure("Thread.start({ println 4 })", "Method calls not allowed on [java.lang.Thread]");
testFailure("import java.util.concurrent.ThreadPoolExecutor;",
"Importing [java.util.concurrent.ThreadPoolExecutor] is not allowed");
testFailure("s = new java.net.URL();", "Expression [ConstructorCallExpression] is not allowed: new java.net.URL()");
}
public void testSuccess(String script) {
logger.info("--> script: " + script);
SearchResponse resp = client().prepareSearch("test")
.setSource("{\"query\": {\"match_all\": {}}," +
"\"sort\":{\"_script\": {\"script\": \""+ script +
"; doc['foo'].value + 2\", \"type\": \"number\", \"lang\": \"groovy\"}}}").get();
assertNoFailures(resp);
assertThat(resp.getHits().getAt(0).getSortValues(), equalTo(new Object[]{7.0}));
}
public void testFailure(String script, String failMessage) {
logger.info("--> script: " + script);
try {
client().prepareSearch("test")
.setSource("{\"query\": {\"match_all\": {}}," +
"\"sort\":{\"_script\": {\"script\": \""+ script +
"; doc['foo'].value + 2\", \"type\": \"number\", \"lang\": \"groovy\"}}}").get();
fail("script: " + script + " failed to be caught be the sandbox!");
} catch (SearchPhaseExecutionException e) {
String msg = ExceptionsHelper.detailedMessage(ExceptionsHelper.unwrapCause(e));
assertThat("script failed, but with incorrect message: " + msg, msg.contains(failMessage), equalTo(true));
}
}
}

View File

@ -205,7 +205,7 @@ public class IndexLookupTests extends ElasticsearchIntegrationTest {
initTestData();
String script = "term = _index['float_payload_field'].get('b'," + includeAllFlag
+ "); payloadSum=0; for (pos : term) {payloadSum = pos.payloadAsInt(0);} return payloadSum;";
+ "); payloadSum=0; for (pos : term) {payloadSum = pos.payloadAsInt(0)}; payloadSum";
// non existing field: sum should be 0
HashMap<String, Object> zeroArray = new HashMap<>();
@ -215,7 +215,7 @@ public class IndexLookupTests extends ElasticsearchIntegrationTest {
checkValueInEachDoc(script, zeroArray, 3);
script = "term = _index['int_payload_field'].get('b'," + includeAllFlag
+ "); payloadSum=0; for (pos : term) {payloadSum = payloadSum + pos.payloadAsInt(0);} return payloadSum;";
+ "); payloadSum=0; for (pos : term) {payloadSum = payloadSum + pos.payloadAsInt(0)}; payloadSum";
// existing field: sums should be as here:
zeroArray.put("1", 5);
@ -263,27 +263,27 @@ public class IndexLookupTests extends ElasticsearchIntegrationTest {
private String createPositionsArrayScriptGetInfoObjectTwice(String term, String flags, String what) {
String script = "term = _index['int_payload_field'].get('" + term + "'," + flags
+ "); array=[]; for (pos : term) {array.add(pos." + what + ")} ;_index['int_payload_field'].get('" + term + "',"
+ "); array=[]; for (pos : term) {array.add(pos." + what + ")}; _index['int_payload_field'].get('" + term + "',"
+ flags + "); array=[]; for (pos : term) {array.add(pos." + what + ")}";
return script;
}
private String createPositionsArrayScriptIterateTwice(String term, String flags, String what) {
String script = "term = _index['int_payload_field'].get('" + term + "'," + flags
+ "); array=[]; for (pos : term) {array.add(pos." + what + ")} array=[]; for (pos : term) {array.add(pos." + what
+ ")} return array;";
+ "); array=[]; for (pos : term) {array.add(pos." + what + ")}; array=[]; for (pos : term) {array.add(pos." + what
+ ")}; array";
return script;
}
private String createPositionsArrayScript(String field, String term, String flags, String what) {
String script = "term = _index['" + field + "'].get('" + term + "'," + flags
+ "); array=[]; for (pos : term) {array.add(pos." + what + ")} return array;";
+ "); array=[]; for (pos : term) {array.add(pos." + what + ")}; array";
return script;
}
private String createPositionsArrayScriptDefaultGet(String field, String term, String what) {
String script = "term = _index['" + field + "']['" + term + "']; array=[]; for (pos : term) {array.add(pos." + what
+ ")} return array;";
+ ")}; array";
return script;
}

View File

@ -0,0 +1,58 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.script.groovy.GroovyScriptEngineService;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.junit.Test;
import static org.hamcrest.Matchers.equalTo;
/**
* Test that a system where the sandbox is disabled while dynamic scripting is
* also disabled does not allow a script to be sent
*/
@ElasticsearchIntegrationTest.ClusterScope(scope=ElasticsearchIntegrationTest.Scope.SUITE)
public class SandboxDisabledTests extends ElasticsearchIntegrationTest {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return ImmutableSettings.settingsBuilder().put(super.nodeSettings(nodeOrdinal))
.put(GroovyScriptEngineService.GROOVY_SCRIPT_SANDBOX_ENABLED, false)
.put(ScriptService.DISABLE_DYNAMIC_SCRIPTING_SETTING, true)
.build();
}
@Test
public void testScriptingDisabledWhileSandboxDisabled() {
client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefresh(true).get();
try {
client().prepareSearch("test")
.setSource("{\"query\": {\"match_all\": {}}," +
"\"sort\":{\"_script\": {\"script\": \"doc['foo'].value + 2\", \"type\": \"number\", \"lang\": \"groovy\"}}}").get();
fail("shards should fail because the sandbox and dynamic scripting are disabled");
} catch (Exception e) {
assertThat(e.getMessage().contains("dynamic scripting for [groovy] disabled"), equalTo(true));
}
}
}

View File

@ -710,7 +710,7 @@ public class DateHistogramTests extends ElasticsearchIntegrationTest {
SearchResponse response = client().prepareSearch("idx")
.addAggregation(dateHistogram("histo")
.field("dates")
.script("new DateTime(_value, DateTimeZone.UTC).plusMonths(1).getMillis()")
.script("new DateTime((long)_value, DateTimeZone.UTC).plusMonths(1).getMillis()")
.interval(DateHistogram.Interval.MONTH)
.subAggregation(max("max")))
.execute().actionGet();

View File

@ -1027,7 +1027,7 @@ public class DoubleTermsTests extends ElasticsearchIntegrationTest {
.setQuery(functionScoreQuery(matchAllQuery()).add(ScoreFunctionBuilders.scriptFunction("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value")))
.addAggregation(terms("terms")
.collectMode(randomFrom(SubAggCollectionMode.values()))
.script("ceil(_doc.score/3)")
.script("ceil(_doc.score()/3)")
).execute().actionGet();
assertSearchResponse(response);

View File

@ -224,7 +224,7 @@ public class AvgTests extends AbstractNumericTests {
public void testScript_MultiValued() throws Exception {
SearchResponse searchResponse = client().prepareSearch("idx")
.setQuery(matchAllQuery())
.addAggregation(avg("avg").script("new double[] { doc['value'].value, doc['value'].value + 1 }"))
.addAggregation(avg("avg").script("[ doc['value'].value, doc['value'].value + 1 ]"))
.execute().actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
@ -239,7 +239,7 @@ public class AvgTests extends AbstractNumericTests {
public void testScript_ExplicitMultiValued() throws Exception {
SearchResponse searchResponse = client().prepareSearch("idx")
.setQuery(matchAllQuery())
.addAggregation(avg("avg").script("new double[] { doc['value'].value, doc['value'].value + 1 }"))
.addAggregation(avg("avg").script("[ doc['value'].value, doc['value'].value + 1 ]"))
.execute().actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
@ -255,7 +255,7 @@ public class AvgTests extends AbstractNumericTests {
public void testScript_MultiValued_WithParams() throws Exception {
SearchResponse searchResponse = client().prepareSearch("idx")
.setQuery(matchAllQuery())
.addAggregation(avg("avg").script("new double[] { doc['value'].value, doc['value'].value + inc }").param("inc", 1))
.addAggregation(avg("avg").script("[ doc['value'].value, doc['value'].value + inc ]").param("inc", 1))
.execute().actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));

View File

@ -366,7 +366,7 @@ public class ExtendedStatsTests extends AbstractNumericTests {
public void testScript_MultiValued_WithParams() throws Exception {
SearchResponse searchResponse = client().prepareSearch("idx")
.setQuery(matchAllQuery())
.addAggregation(extendedStats("stats").script("new double[] { doc['value'].value, doc['value'].value - dec }").param("dec", 1))
.addAggregation(extendedStats("stats").script("[ doc['value'].value, doc['value'].value - dec ]").param("dec", 1))
.execute().actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));

View File

@ -254,7 +254,7 @@ public class MaxTests extends AbstractNumericTests {
public void testScript_MultiValued_WithParams() throws Exception {
SearchResponse searchResponse = client().prepareSearch("idx")
.setQuery(matchAllQuery())
.addAggregation(max("max").script("new double[] { doc['value'].value, doc['value'].value + inc }").param("inc", 1))
.addAggregation(max("max").script("[ doc['value'].value, doc['value'].value + inc ]").param("inc", 1))
.execute().actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));

View File

@ -339,7 +339,7 @@ public class StatsTests extends AbstractNumericTests {
public void testScript_MultiValued_WithParams() throws Exception {
SearchResponse searchResponse = client().prepareSearch("idx")
.setQuery(matchAllQuery())
.addAggregation(stats("stats").script("new double[] { doc['value'].value, doc['value'].value - dec }").param("dec", 1))
.addAggregation(stats("stats").script("[ doc['value'].value, doc['value'].value - dec ]").param("dec", 1))
.execute().actionGet();
assertShardExecutionState(searchResponse, 0);

View File

@ -179,7 +179,7 @@ public class SumTests extends AbstractNumericTests {
public void testScript_MultiValued() throws Exception {
SearchResponse searchResponse = client().prepareSearch("idx")
.setQuery(matchAllQuery())
.addAggregation(sum("sum").script("new double[] { doc['value'].value, doc['value'].value + 1 }"))
.addAggregation(sum("sum").script("[ doc['value'].value, doc['value'].value + 1 ]"))
.execute().actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
@ -194,7 +194,7 @@ public class SumTests extends AbstractNumericTests {
public void testScript_ExplicitMultiValued() throws Exception {
SearchResponse searchResponse = client().prepareSearch("idx")
.setQuery(matchAllQuery())
.addAggregation(sum("sum").script("new double[] { doc['value'].value, doc['value'].value + 1 }"))
.addAggregation(sum("sum").script("[ doc['value'].value, doc['value'].value + 1 ]"))
.execute().actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));
@ -209,7 +209,7 @@ public class SumTests extends AbstractNumericTests {
public void testScript_MultiValued_WithParams() throws Exception {
SearchResponse searchResponse = client().prepareSearch("idx")
.setQuery(matchAllQuery())
.addAggregation(sum("sum").script("new double[] { doc['value'].value, doc['value'].value + inc }").param("inc", 1))
.addAggregation(sum("sum").script("[ doc['value'].value, doc['value'].value + inc ]").param("inc", 1))
.execute().actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(10l));

View File

@ -22,6 +22,8 @@ package org.elasticsearch.search.scriptfilter;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.script.groovy.GroovyScriptEngineService;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.junit.Test;
@ -37,8 +39,14 @@ import static org.hamcrest.Matchers.equalTo;
/**
*
*/
@ElasticsearchIntegrationTest.ClusterScope(scope=ElasticsearchIntegrationTest.Scope.SUITE)
public class ScriptFilterSearchTests extends ElasticsearchIntegrationTest {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return ImmutableSettings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)).put(GroovyScriptEngineService.GROOVY_SCRIPT_SANDBOX_ENABLED, false).build();
}
@Test
public void testCustomScriptBoost() throws Exception {
createIndex("test");

View File

@ -687,7 +687,7 @@ public class SimpleSortTests extends ElasticsearchIntegrationTest {
// test the long values
SearchResponse searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addScriptField("min", "var retval = Long.MAX_VALUE; for (v : doc['lvalue'].values){ retval = Math.min(v, retval);} return retval;")
.addScriptField("min", "retval = Long.MAX_VALUE; for (v : doc['lvalue'].values){ retval = min(v, retval) }; retval")
.addSort("ord", SortOrder.ASC).setSize(10)
.execute().actionGet();
@ -700,7 +700,7 @@ public class SimpleSortTests extends ElasticsearchIntegrationTest {
// test the double values
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addScriptField("min", "var retval = Double.MAX_VALUE; for (v : doc['dvalue'].values){ retval = Math.min(v, retval);} return retval;")
.addScriptField("min", "retval = Double.MAX_VALUE; for (v : doc['dvalue'].values){ retval = min(v, retval) }; retval")
.addSort("ord", SortOrder.ASC).setSize(10)
.execute().actionGet();
@ -714,7 +714,7 @@ public class SimpleSortTests extends ElasticsearchIntegrationTest {
// test the string values
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addScriptField("min", "var retval = Integer.MAX_VALUE; for (v : doc['svalue'].values){ retval = Math.min(Integer.parseInt(v), retval);} return retval;")
.addScriptField("min", "retval = Integer.MAX_VALUE; for (v : doc['svalue'].values){ retval = min(Integer.parseInt(v), retval) }; retval")
.addSort("ord", SortOrder.ASC).setSize(10)
.execute().actionGet();
@ -728,7 +728,7 @@ public class SimpleSortTests extends ElasticsearchIntegrationTest {
// test the geopoint values
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addScriptField("min", "var retval = Double.MAX_VALUE; for (v : doc['gvalue'].values){ retval = Math.min(v.lon, retval);} return retval;")
.addScriptField("min", "retval = Double.MAX_VALUE; for (v : doc['gvalue'].values){ retval = min(v.lon, retval) }; retval")
.addSort("ord", SortOrder.ASC).setSize(10)
.execute().actionGet();

View File

@ -20,6 +20,9 @@
package org.elasticsearch.search.timeout;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.script.groovy.GroovyScriptEngineService;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.junit.Test;
@ -30,16 +33,17 @@ import static org.hamcrest.Matchers.equalTo;
/**
*/
@ElasticsearchIntegrationTest.ClusterScope(scope=ElasticsearchIntegrationTest.Scope.SUITE)
public class SearchTimeoutTests extends ElasticsearchIntegrationTest {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return ImmutableSettings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)).put(GroovyScriptEngineService.GROOVY_SCRIPT_SANDBOX_ENABLED, false).build();
}
@Test
public void simpleTimeoutTest() throws Exception {
createIndex("test");
for (int i = 0; i < 10; i++) {
client().prepareIndex("test", "type", Integer.toString(i)).setSource("field", "value").execute().actionGet();
}
refresh();
client().prepareIndex("test", "type", "1").setSource("field", "value").setRefresh(true).execute().actionGet();
SearchResponse searchResponse = client().prepareSearch("test")
.setTimeout("10ms")