honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/search-request-refactoring

This commit is contained in:
javanna 2015-10-06 14:28:31 +02:00 committed by Luca Cavanna
parent a3a2432ebd 7b431ecc16
commit f89de33548
40 changed files with 397 additions and 345 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
core/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java
View File

@ -45,9 +45,16 @@ public final class BootstrapInfo {
}
/**
* Returns true if secure computing mode is enabled (linux/amd64 only)
* Returns true if secure computing mode is enabled (linux/amd64, OS X only)
*/
public static boolean isSeccompInstalled() {
return Natives.isSeccompInstalled();
}
/**
* codebase location for untrusted scripts (provide some additional safety)
* <p>
* This is not a full URL, just a path.
*/
public static final String UNTRUSTED_CODEBASE = "/untrusted";
}

32
core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java
View File

@ -26,29 +26,27 @@ import java.net.URL;
import java.security.CodeSource;
import java.security.Permission;
import java.security.PermissionCollection;
import java.security.Permissions;
import java.security.Policy;
import java.security.ProtectionDomain;
import java.security.URIParameter;
import java.util.PropertyPermission;
/** custom policy for union of static and dynamic permissions */
final class ESPolicy extends Policy {
/** template policy file, the one used in tests */
static final String POLICY_RESOURCE = "security.policy";
/** limited policy for groovy scripts */
static final String GROOVY_RESOURCE = "groovy.policy";
/** limited policy for scripts */
static final String UNTRUSTED_RESOURCE = "untrusted.policy";
final Policy template;
final Policy groovy;
final Policy untrusted;
final PermissionCollection dynamic;
public ESPolicy(PermissionCollection dynamic) throws Exception {
URI policyUri = getClass().getResource(POLICY_RESOURCE).toURI();
URI groovyUri = getClass().getResource(GROOVY_RESOURCE).toURI();
URI untrustedUri = getClass().getResource(UNTRUSTED_RESOURCE).toURI();
this.template = Policy.getInstance("JavaPolicy", new URIParameter(policyUri));
this.groovy = Policy.getInstance("JavaPolicy", new URIParameter(groovyUri));
this.untrusted = Policy.getInstance("JavaPolicy", new URIParameter(untrustedUri));
this.dynamic = dynamic;
}
@ -56,15 +54,17 @@ final class ESPolicy extends Policy {
public boolean implies(ProtectionDomain domain, Permission permission) {
CodeSource codeSource = domain.getCodeSource();
// codesource can be null when reducing privileges via doPrivileged()
if (codeSource != null) {
URL location = codeSource.getLocation();
// location can be null... ??? nobody knows
// https://bugs.openjdk.java.net/browse/JDK-8129972
if (location != null) {
// run groovy scripts with no permissions (except logging property)
if ("/groovy/script".equals(location.getFile())) {
return groovy.implies(domain, permission);
}
if (codeSource == null) {
return false;
}
URL location = codeSource.getLocation();
// location can be null... ??? nobody knows
// https://bugs.openjdk.java.net/browse/JDK-8129972
if (location != null) {
// run scripts with limited permissions
if (BootstrapInfo.UNTRUSTED_CODEBASE.equals(location.getFile())) {
return untrusted.implies(domain, permission);
}
}

6
core/src/main/java/org/elasticsearch/index/IndexModule.java
View File

@ -21,8 +21,11 @@ package org.elasticsearch.index;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.util.Providers;
import org.elasticsearch.index.aliases.IndexAliasesService;
import org.elasticsearch.index.engine.EngineFactory;
import org.elasticsearch.index.engine.InternalEngineFactory;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.shard.IndexSearcherWrapper;
/**
@ -44,6 +47,9 @@ public class IndexModule extends AbstractModule {
}
bind(IndexService.class).asEagerSingleton();
bind(IndexServicesProvider.class).asEagerSingleton();
bind(MapperService.class).asEagerSingleton();
bind(IndexAliasesService.class).asEagerSingleton();
bind(IndexFieldDataService.class).asEagerSingleton();
}

33
core/src/main/java/org/elasticsearch/index/aliases/IndexAliasesServiceModule.java
View File

@ -1,33 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.aliases;
import org.elasticsearch.common.inject.AbstractModule;
/**
*
*/
public class IndexAliasesServiceModule extends AbstractModule {
@Override
protected void configure() {
bind(IndexAliasesService.class).asEagerSingleton();
}
}

38
core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataModule.java
View File

@ -1,38 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.settings.Settings;
/**
*/
public class IndexFieldDataModule extends AbstractModule {
private final Settings settings;
public IndexFieldDataModule(Settings settings) {
this.settings = settings;
}
@Override
protected void configure() {
bind(IndexFieldDataService.class).asEagerSingleton();
}
}

33
core/src/main/java/org/elasticsearch/index/mapper/MapperServiceModule.java
View File

@ -1,33 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper;
import org.elasticsearch.common.inject.AbstractModule;
/**
*
*/
public class MapperServiceModule extends AbstractModule {
@Override
protected void configure() {
bind(MapperService.class).asEagerSingleton();
}
}

6
core/src/main/java/org/elasticsearch/indices/IndicesService.java
View File

@ -53,18 +53,15 @@ import org.elasticsearch.index.IndexNameModule;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.LocalNodeIdModule;
import org.elasticsearch.index.aliases.IndexAliasesServiceModule;
import org.elasticsearch.index.analysis.AnalysisModule;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.cache.IndexCache;
import org.elasticsearch.index.cache.IndexCacheModule;
import org.elasticsearch.index.fielddata.IndexFieldDataModule;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.flush.FlushStats;
import org.elasticsearch.index.get.GetStats;
import org.elasticsearch.index.indexing.IndexingStats;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.MapperServiceModule;
import org.elasticsearch.index.merge.MergeStats;
import org.elasticsearch.index.query.IndexQueryParserService;
import org.elasticsearch.index.recovery.RecoveryStats;
@ -343,9 +340,6 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
modules.add(new AnalysisModule(indexSettings, indicesAnalysisService));
modules.add(new SimilarityModule(indexSettings));
modules.add(new IndexCacheModule(indexSettings));
modules.add(new IndexFieldDataModule(indexSettings));
modules.add(new MapperServiceModule());
modules.add(new IndexAliasesServiceModule());
modules.add(new IndexModule());
pluginsService.processModules(modules);

5
core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
View File

@ -98,11 +98,6 @@ public class NativeScriptEngineService extends AbstractComponent implements Scri
return executable(compiledScript, vars).run();
}
@Override
public Object unwrap(Object value) {
return value;
}
@Override
public void close() {
}

2
core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
View File

@ -44,8 +44,6 @@ public interface ScriptEngineService extends Closeable {
Object execute(CompiledScript compiledScript, Map<String, Object> vars);
Object unwrap(Object value);
/**
* Handler method called when a script is removed from the Guava cache.
*

5
core/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
View File

@ -138,11 +138,6 @@ public class MustacheScriptEngineService extends AbstractComponent implements Sc
throw new UnsupportedOperationException();
}
@Override
public Object unwrap(Object value) {
return value;
}
@Override
public void close() {
// Nothing to do here

4
core/src/main/resources/org/elasticsearch/bootstrap/security.policy
View File

@ -69,8 +69,8 @@ grant codeBase "${es.security.plugin.lang-groovy}" {
permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect";
// needed by GroovyScriptEngineService to close its classloader (why?)
permission java.lang.RuntimePermission "closeClassLoader";
// Allow executing groovy scripts with codesource of /groovy/script
permission groovy.security.GroovyCodeSourcePermission "/groovy/script";
// Allow executing groovy scripts with codesource of /untrusted
permission groovy.security.GroovyCodeSourcePermission "/untrusted";
};
grant codeBase "${es.security.plugin.lang-javascript}" {

4
core/src/main/resources/org/elasticsearch/bootstrap/groovy.policy → core/src/main/resources/org/elasticsearch/bootstrap/untrusted.policy
View File

@ -18,8 +18,8 @@
*/
/*
* Limited security policy for groovy scripts.
* This is what is needed for its invokeDynamic functionality to work.
* Limited security policy for scripts.
* This is what is needed for invokeDynamic functionality to work.
*/
grant {

9
core/src/test/java/org/elasticsearch/bootstrap/ESPolicyTests.java
View File

@ -24,7 +24,9 @@ import org.elasticsearch.test.ESTestCase;
import java.io.FilePermission;
import java.security.AccessControlContext;
import java.security.AccessController;
import java.security.AllPermission;
import java.security.CodeSource;
import java.security.Permission;
import java.security.PermissionCollection;
import java.security.Permissions;
import java.security.PrivilegedAction;
@ -48,8 +50,13 @@ public class ESPolicyTests extends ESTestCase {
*/
public void testNullCodeSource() throws Exception {
assumeTrue("test cannot run with security manager", System.getSecurityManager() == null);
// create a policy with AllPermission
Permission all = new AllPermission();
PermissionCollection allCollection = all.newPermissionCollection();
allCollection.add(all);
ESPolicy policy = new ESPolicy(allCollection);
// restrict ourselves to NoPermission
PermissionCollection noPermissions = new Permissions();
ESPolicy policy = new ESPolicy(noPermissions);
assertFalse(policy.implies(new ProtectionDomain(null, noPermissions), new FilePermission("foo", "read")));
}

15
core/src/test/java/org/elasticsearch/bootstrap/MockPluginPolicy.java
View File

@ -35,7 +35,6 @@ import java.security.ProtectionDomain;
import java.security.cert.Certificate;
import java.util.Collections;
import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
/**
@ -99,18 +98,24 @@ final class MockPluginPolicy extends Policy {
excludedSources.add(RandomizedRunner.class.getProtectionDomain().getCodeSource());
// junit library
excludedSources.add(Assert.class.getProtectionDomain().getCodeSource());
// groovy scripts
excludedSources.add(new CodeSource(new URL("file:/groovy/script"), (Certificate[])null));
// scripts
excludedSources.add(new CodeSource(new URL("file:" + BootstrapInfo.UNTRUSTED_CODEBASE), (Certificate[])null));
Loggers.getLogger(getClass()).debug("Apply permissions [{}] excluding codebases [{}]", extraPermissions, excludedSources);
}
@Override
public boolean implies(ProtectionDomain domain, Permission permission) {
CodeSource codeSource = domain.getCodeSource();
// codesource can be null when reducing privileges via doPrivileged()
if (codeSource == null) {
return false;
}
if (standardPolicy.implies(domain, permission)) {
return true;
} else if (excludedSources.contains(domain.getCodeSource()) == false &&
Objects.toString(domain.getCodeSource()).contains("test-classes") == false) {
} else if (excludedSources.contains(codeSource) == false &&
codeSource.toString().contains("test-classes") == false) {
return extraPermissions.implies(permission);
} else {
return false;

18
plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/AliasRoutingTests.java → core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java
View File

@ -17,22 +17,15 @@
* under the License.
*/
package org.elasticsearch.messy.tests;
package org.elasticsearch.routing;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.groovy.GroovyPlugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Test;
import java.util.Collection;
import java.util.Collections;
import static org.elasticsearch.cluster.metadata.AliasAction.newAddAliasAction;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
@ -40,13 +33,8 @@ import static org.hamcrest.Matchers.equalTo;
/**
*
*/
public class AliasRoutingTests extends ESIntegTestCase {
public class AliasRoutingIT extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Collections.singleton(GroovyPlugin.class);
}
@Override
protected int minimumNumberOfShards() {
return 2;
@ -77,7 +65,7 @@ public class AliasRoutingTests extends ESIntegTestCase {
logger.info("--> updating with id [1] and routing through alias");
client().prepareUpdate("alias0", "type1", "1")
.setUpsert(XContentFactory.jsonBuilder().startObject().field("field", 1).endObject())
.setScript(new Script("ctx._source.field = 'value2'", ScriptService.ScriptType.INLINE, null, null))
.setDoc("field", "value2")
.execute().actionGet();
for (int i = 0; i < 5; i++) {
assertThat(client().prepareGet("alias0", "type1", "1").execute().actionGet().isExists(), equalTo(true));

5
core/src/test/java/org/elasticsearch/script/MockScriptEngine.java
View File

@ -108,11 +108,6 @@ public class MockScriptEngine implements ScriptEngineService {
return null;
}
@Override
public Object unwrap(Object value) {
return null;
}
@Override
public void scriptRemoved(@Nullable CompiledScript script) {
}

5
core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
View File

@ -290,11 +290,6 @@ public class ScriptModesTests extends ESTestCase {
return null;
}
@Override
public Object unwrap(Object value) {
return null;
}
@Override
public void close() {

5
core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
View File

@ -501,11 +501,6 @@ public class ScriptServiceTests extends ESTestCase {
return null;
}
@Override
public Object unwrap(Object value) {
return null;
}
@Override
public void close() {

2
core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
View File

@ -55,7 +55,6 @@ import org.elasticsearch.index.IndexNameModule;
import org.elasticsearch.index.analysis.AnalysisModule;
import org.elasticsearch.index.cache.IndexCacheModule;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.MapperServiceModule;
import org.elasticsearch.index.query.AbstractQueryTestCase;
import org.elasticsearch.index.query.IndexQueryParserService;
import org.elasticsearch.index.query.QueryBuilders;
@ -167,7 +166,6 @@ public class SearchSourceBuilderTests extends ESTestCase {
new EnvironmentModule(new Environment(settings)),
new SettingsModule(settings),
new ThreadPoolModule(new ThreadPool(settings)),
new MapperServiceModule(),
new IndicesModule(settings) {
@Override
public void configure() {

44
plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/QueryRescorerTests.java → core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package org.elasticsearch.messy.tests;
package org.elasticsearch.search.functionscore;
@ -35,9 +35,6 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.Operator;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.groovy.GroovyPlugin;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.rescore.RescoreBuilder;
@ -46,8 +43,6 @@ import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Test;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
@ -59,12 +54,7 @@ import static org.hamcrest.Matchers.*;
/**
*
*/
public class QueryRescorerTests extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Collections.singleton(GroovyPlugin.class);
}
public class QueryRescorerIT extends ESIntegTestCase {
@Test
@AwaitsFix(bugUrl = "Need to fix default window size for rescorers so that they are applied")
@ -604,11 +594,11 @@ public class QueryRescorerTests extends ESIntegTestCase {
QueryBuilders.boolQuery()
.disableCoord(true)
.should(QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("field1", intToEnglish[0]),
ScoreFunctionBuilders.scriptFunction(new Script("5.0f"))).boostMode(CombineFunction.REPLACE))
ScoreFunctionBuilders.weightFactorFunction(5.0f)).boostMode(CombineFunction.REPLACE))
.should(QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("field1", intToEnglish[1]),
ScoreFunctionBuilders.scriptFunction(new Script("7.0f"))).boostMode(CombineFunction.REPLACE))
ScoreFunctionBuilders.weightFactorFunction(7.0f)).boostMode(CombineFunction.REPLACE))
.should(QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("field1", intToEnglish[3]),
ScoreFunctionBuilders.scriptFunction(new Script("0.0f"))).boostMode(CombineFunction.REPLACE)))
ScoreFunctionBuilders.weightFactorFunction(0.0f)).boostMode(CombineFunction.REPLACE)))
.setQueryWeight(primaryWeight)
.setRescoreQueryWeight(secondaryWeight);
@ -622,17 +612,17 @@ public class QueryRescorerTests extends ESIntegTestCase {
.setQuery(QueryBuilders.boolQuery()
.disableCoord(true)
.should(QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("field1", intToEnglish[0]),
ScoreFunctionBuilders.scriptFunction(new Script("2.0f"))).boostMode(CombineFunction.REPLACE))
ScoreFunctionBuilders.weightFactorFunction(2.0f)).boostMode(CombineFunction.REPLACE))
.should(QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("field1", intToEnglish[1]),
ScoreFunctionBuilders.scriptFunction(new Script("3.0f"))).boostMode(CombineFunction.REPLACE))
ScoreFunctionBuilders.weightFactorFunction(3.0f)).boostMode(CombineFunction.REPLACE))
.should(QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("field1", intToEnglish[2]),
ScoreFunctionBuilders.scriptFunction(new Script("5.0f"))).boostMode(CombineFunction.REPLACE))
ScoreFunctionBuilders.weightFactorFunction(5.0f)).boostMode(CombineFunction.REPLACE))
.should(QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("field1", intToEnglish[3]),
ScoreFunctionBuilders.scriptFunction(new Script("0.2f"))).boostMode(CombineFunction.REPLACE)))
.setFrom(0)
.setSize(10)
.setRescorer(rescoreQuery)
.setRescoreWindow(50).execute().actionGet();
ScoreFunctionBuilders.weightFactorFunction(0.2f)).boostMode(CombineFunction.REPLACE)))
.setFrom(0)
.setSize(10)
.setRescorer(rescoreQuery)
.setRescoreWindow(50).execute().actionGet();
assertHitCount(rescored, 4);
@ -688,10 +678,10 @@ public class QueryRescorerTests extends ESIntegTestCase {
int numDocs = indexRandomNumbers("keyword", 1, true);
QueryRescorer eightIsGreat = RescoreBuilder.queryRescorer(
QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("field1", English.intToEnglish(8)),
ScoreFunctionBuilders.scriptFunction(new Script("1000.0f"))).boostMode(CombineFunction.REPLACE)).setScoreMode("total");
ScoreFunctionBuilders.weightFactorFunction(1000.0f)).boostMode(CombineFunction.REPLACE)).setScoreMode("total");
QueryRescorer sevenIsBetter = RescoreBuilder.queryRescorer(
QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("field1", English.intToEnglish(7)),
ScoreFunctionBuilders.scriptFunction(new Script("10000.0f"))).boostMode(CombineFunction.REPLACE))
ScoreFunctionBuilders.weightFactorFunction(10000.0f)).boostMode(CombineFunction.REPLACE))
.setScoreMode("total");
// First set the rescore window large enough that both rescores take effect
@ -708,10 +698,10 @@ public class QueryRescorerTests extends ESIntegTestCase {
// Now use one rescore to drag the number we're looking for into the window of another
QueryRescorer ninetyIsGood = RescoreBuilder.queryRescorer(
QueryBuilders.functionScoreQuery(QueryBuilders.queryStringQuery("*ninety*"), ScoreFunctionBuilders.scriptFunction(new Script("1000.0f")))
QueryBuilders.functionScoreQuery(QueryBuilders.queryStringQuery("*ninety*"), ScoreFunctionBuilders.weightFactorFunction(1000.0f))
.boostMode(CombineFunction.REPLACE)).setScoreMode("total");
QueryRescorer oneToo = RescoreBuilder.queryRescorer(
QueryBuilders.functionScoreQuery(QueryBuilders.queryStringQuery("*one*"), ScoreFunctionBuilders.scriptFunction(new Script("1000.0f")))
QueryBuilders.functionScoreQuery(QueryBuilders.queryStringQuery("*one*"), ScoreFunctionBuilders.weightFactorFunction(1000.0f))
.boostMode(CombineFunction.REPLACE)).setScoreMode("total");
request.clearRescorers().addRescorer(ninetyIsGood).addRescorer(oneToo, 10);
response = request.setSize(2).get();

31
docs/plugins/discovery-ec2.asciidoc
View File

@ -165,6 +165,37 @@ The following are a list of settings (prefixed with `discovery.ec2`) that can fu
Defaults to `3s`. If no unit like `ms`, `s` or `m` is specified,
milliseconds are used.
[IMPORTANT]
.Binding the network host
==============================================
It's important to define `network.host` as by default it's bound to `localhost`.
You can use {ref}/modules-network.html[core network host settings] or
<<discovery-ec2-network-host,ec2 specific host settings>>:
==============================================
[[discovery-ec2-network-host]]
===== EC2 Network Host
When the `discovery-ec2` plugin is installed, the following are also allowed
as valid network host settings:
[cols="<,<",options="header",]
|==================================================================
|EC2 Host Value |Description
|`_ec2:privateIpv4_` |The private IP address (ipv4) of the machine.
|`_ec2:privateDns_` |The private host of the machine.
|`_ec2:publicIpv4_` |The public IP address (ipv4) of the machine.
|`_ec2:publicDns_` |The public host of the machine.
|`_ec2:privateIp_` |equivalent to _ec2:privateIpv4_.
|`_ec2:publicIp_` |equivalent to _ec2:publicIpv4_.
|`_ec2_` |equivalent to _ec2:privateIpv4_.
|==================================================================
[[discovery-ec2-permissions]]
===== Recommended EC2 Permissions

16
docs/reference/modules/network.asciidoc
View File

@ -51,20 +51,8 @@ provided network interface. For example `_en0:ipv4_`.
provided network interface. For example `_en0:ipv6_`.
|=======================================================================
When the `discovery-ec2` plugin is installed, the following are also allowed
as valid network host settings:
[cols="<,<",options="header",]
|==================================================================
|EC2 Host Value |Description
|`_ec2:privateIpv4_` |The private IP address (ipv4) of the machine.
|`_ec2:privateDns_` |The private host of the machine.
|`_ec2:publicIpv4_` |The public IP address (ipv4) of the machine.
|`_ec2:publicDns_` |The public host of the machine.
|`_ec2_` |Less verbose option for the private ip address.
|`_ec2:privateIp_` |Less verbose option for the private ip address.
|`_ec2:publicIp_` |Less verbose option for the public ip address.
|==================================================================
When the `discovery-ec2` plugin is installed, you can use
{plugins}/discovery-ec2-discovery.html#discovery-ec2-network-host[ec2 specific host settings].
[float]
[[tcp-settings]]

5
plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
View File

@ -242,11 +242,6 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
return expressionExecutableScript.run();
}
@Override
public Object unwrap(Object value) {
return value;
}
@Override
public void close() {}

17
plugins/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.script.groovy;
import groovy.lang.Binding;
import groovy.lang.GroovyClassLoader;
import groovy.lang.GroovyCodeSource;
import groovy.lang.Script;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.Scorer;
@ -36,6 +37,7 @@ import org.codehaus.groovy.control.SourceUnit;
import org.codehaus.groovy.control.customizers.CompilationCustomizer;
import org.codehaus.groovy.control.customizers.ImportCustomizer;
import org.elasticsearch.SpecialPermission;
import org.elasticsearch.bootstrap.BootstrapInfo;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.hash.MessageDigests;
@ -168,7 +170,15 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
if (sm != null) {
sm.checkPermission(new SpecialPermission());
}
return loader.parseClass(script, MessageDigests.toHexString(MessageDigests.sha1().digest(script.getBytes(StandardCharsets.UTF_8))));
String fake = MessageDigests.toHexString(MessageDigests.sha1().digest(script.getBytes(StandardCharsets.UTF_8)));
// same logic as GroovyClassLoader.parseClass() but with a different codesource string:
GroovyCodeSource gcs = AccessController.doPrivileged(new PrivilegedAction<GroovyCodeSource>() {
public GroovyCodeSource run() {
return new GroovyCodeSource(script, fake, BootstrapInfo.UNTRUSTED_CODEBASE);
}
});
gcs.setCachable(false);
return loader.parseClass(gcs);
} catch (Throwable e) {
if (logger.isTraceEnabled()) {
logger.trace("exception compiling Groovy script:", e);
@ -248,11 +258,6 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
}
}
@Override
public Object unwrap(Object value) {
return value;
}
public static final class GroovyScript implements ExecutableScript, LeafSearchScript {
private final CompiledScript compiledScript;

89
plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java
View File

@ -57,45 +57,16 @@ import org.hamcrest.Matchers;
import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.*;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction;
import static org.elasticsearch.index.query.QueryBuilders.*;
import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction;
import static org.elasticsearch.search.sort.SortBuilders.fieldSort;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFirstHit;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertOrderedSearchHits;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSecondHit;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSortValues;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
import static org.hamcrest.Matchers.closeTo;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.lessThan;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*;
import static org.hamcrest.Matchers.*;
/**
@ -107,7 +78,7 @@ public class SimpleSortTests extends ESIntegTestCase {
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Collections.singleton(GroovyPlugin.class);
}
@TestLogging("action.search.type:TRACE")
@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elasticsearch/elasticsearch/issues/9421")
public void testIssue8226() {
@ -391,7 +362,7 @@ public class SimpleSortTests extends ESIntegTestCase {
SearchResponse searchResponse = client()
.prepareSearch("test")
.setQuery(
QueryBuilders.functionScoreQuery(matchAllQuery(), ScoreFunctionBuilders.scriptFunction(new Script("_source.field"))))
QueryBuilders.functionScoreQuery(matchAllQuery(), ScoreFunctionBuilders.fieldValueFactorFunction("field")))
.execute().actionGet();
assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(1).score(), Matchers.lessThan(searchResponse.getHits().getAt(0).score()));
@ -402,7 +373,7 @@ public class SimpleSortTests extends ESIntegTestCase {
searchResponse = client()
.prepareSearch("test")
.setQuery(
QueryBuilders.functionScoreQuery(matchAllQuery(), ScoreFunctionBuilders.scriptFunction(new Script("_source.field"))))
QueryBuilders.functionScoreQuery(matchAllQuery(), ScoreFunctionBuilders.fieldValueFactorFunction("field")))
.addSort("_score", SortOrder.DESC).execute().actionGet();
assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(1).score(), Matchers.lessThan(searchResponse.getHits().getAt(0).score()));
@ -413,7 +384,7 @@ public class SimpleSortTests extends ESIntegTestCase {
searchResponse = client()
.prepareSearch("test")
.setQuery(
QueryBuilders.functionScoreQuery(matchAllQuery(), ScoreFunctionBuilders.scriptFunction(new Script("_source.field"))))
QueryBuilders.functionScoreQuery(matchAllQuery(), ScoreFunctionBuilders.fieldValueFactorFunction("field")))
.addSort("_score", SortOrder.DESC).execute().actionGet();
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3"));
assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("2"));
@ -433,7 +404,7 @@ public class SimpleSortTests extends ESIntegTestCase {
refresh();
SearchResponse searchResponse = client().prepareSearch("test")
.setQuery(functionScoreQuery(matchAllQuery(), scriptFunction(new Script("_source.field")))).execute().actionGet();
.setQuery(functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("field"))).execute().actionGet();
assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(1).score(), Matchers.lessThan(searchResponse.getHits().getAt(0).score()));
assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("2"));
@ -441,7 +412,7 @@ public class SimpleSortTests extends ESIntegTestCase {
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3"));
searchResponse = client().prepareSearch("test")
.setQuery(functionScoreQuery(matchAllQuery(), scriptFunction(new Script("_source.field"))))
.setQuery(functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("field")))
.addSort("_score", SortOrder.DESC).execute().actionGet();
assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(1).score(), Matchers.lessThan(searchResponse.getHits().getAt(0).score()));
@ -450,7 +421,7 @@ public class SimpleSortTests extends ESIntegTestCase {
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3"));
searchResponse = client().prepareSearch("test")
.setQuery(functionScoreQuery(matchAllQuery(), scriptFunction(new Script("_source.field"))))
.setQuery(functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("field")))
.addSort("_score", SortOrder.DESC).execute().actionGet();
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3"));
assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("2"));
@ -907,9 +878,9 @@ public class SimpleSortTests extends ESIntegTestCase {
assertNoFailures(searchResponse);
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat((String) searchResponse.getHits().getAt(0).field("id").value(), equalTo("1"));
assertThat((String) searchResponse.getHits().getAt(1).field("id").value(), equalTo("3"));
assertThat((String) searchResponse.getHits().getAt(2).field("id").value(), equalTo("2"));
assertThat(searchResponse.getHits().getAt(0).field("id").value(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(1).field("id").value(), equalTo("3"));
assertThat(searchResponse.getHits().getAt(2).field("id").value(), equalTo("2"));
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
@ -920,9 +891,9 @@ public class SimpleSortTests extends ESIntegTestCase {
assertNoFailures(searchResponse);
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat((String) searchResponse.getHits().getAt(0).field("id").value(), equalTo("1"));
assertThat((String) searchResponse.getHits().getAt(1).field("id").value(), equalTo("3"));
assertThat((String) searchResponse.getHits().getAt(2).field("id").value(), equalTo("2"));
assertThat(searchResponse.getHits().getAt(0).field("id").value(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(1).field("id").value(), equalTo("3"));
assertThat(searchResponse.getHits().getAt(2).field("id").value(), equalTo("2"));
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
@ -939,9 +910,9 @@ public class SimpleSortTests extends ESIntegTestCase {
assertThat(searchResponse.getFailedShards(), equalTo(0));
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat((String) searchResponse.getHits().getAt(0).field("id").value(), equalTo("3"));
assertThat((String) searchResponse.getHits().getAt(1).field("id").value(), equalTo("1"));
assertThat((String) searchResponse.getHits().getAt(2).field("id").value(), equalTo("2"));
assertThat(searchResponse.getHits().getAt(0).field("id").value(), equalTo("3"));
assertThat(searchResponse.getHits().getAt(1).field("id").value(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(2).field("id").value(), equalTo("2"));
// a query with docs just with null values
searchResponse = client().prepareSearch()
@ -959,7 +930,7 @@ public class SimpleSortTests extends ESIntegTestCase {
assertThat(searchResponse.getFailedShards(), equalTo(0));
assertThat(searchResponse.getHits().getTotalHits(), equalTo(1l));
assertThat((String) searchResponse.getHits().getAt(0).field("id").value(), equalTo("2"));
assertThat(searchResponse.getHits().getAt(0).field("id").value(), equalTo("2"));
}
@Test
@ -1782,32 +1753,32 @@ public class SimpleSortTests extends ESIntegTestCase {
.addSort(new GeoDistanceSortBuilder("location").points(q).sortMode("min").order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
.execute().actionGet();
assertOrderedSearchHits(searchResponse, "d1", "d2");
assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 2, 3, 2, DistanceUnit.KILOMETERS)));
assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 1, 5, 1, DistanceUnit.KILOMETERS)));
assertThat(searchResponse.getHits().getAt(0).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 2, 3, 2, DistanceUnit.KILOMETERS)));
assertThat(searchResponse.getHits().getAt(1).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 1, 5, 1, DistanceUnit.KILOMETERS)));
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addSort(new GeoDistanceSortBuilder("location").points(q).sortMode("min").order(SortOrder.DESC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
.execute().actionGet();
assertOrderedSearchHits(searchResponse, "d2", "d1");
assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 1, 5, 1, DistanceUnit.KILOMETERS)));
assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 2, 3, 2, DistanceUnit.KILOMETERS)));
assertThat(searchResponse.getHits().getAt(0).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 1, 5, 1, DistanceUnit.KILOMETERS)));
assertThat(searchResponse.getHits().getAt(1).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 2, 3, 2, DistanceUnit.KILOMETERS)));
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addSort(new GeoDistanceSortBuilder("location").points(q).sortMode("max").order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
.execute().actionGet();
assertOrderedSearchHits(searchResponse, "d1", "d2");
assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 2, 4, 1, DistanceUnit.KILOMETERS)));
assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 1, 6, 2, DistanceUnit.KILOMETERS)));
assertThat(searchResponse.getHits().getAt(0).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 2, 4, 1, DistanceUnit.KILOMETERS)));
assertThat(searchResponse.getHits().getAt(1).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 1, 6, 2, DistanceUnit.KILOMETERS)));
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addSort(new GeoDistanceSortBuilder("location").points(q).sortMode("max").order(SortOrder.DESC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
.execute().actionGet();
assertOrderedSearchHits(searchResponse, "d2", "d1");
assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 1, 6, 2, DistanceUnit.KILOMETERS)));
assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 2, 4, 1, DistanceUnit.KILOMETERS)));
assertThat(searchResponse.getHits().getAt(0).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 1, 6, 2, DistanceUnit.KILOMETERS)));
assertThat(searchResponse.getHits().getAt(1).getSortValues()[0], equalTo(GeoDistance.PLANE.calculate(2, 2, 4, 1, DistanceUnit.KILOMETERS)));
}
protected void createShuffeldJSONArray(XContentBuilder builder, GeoPoint[] pointsArray) throws IOException {

3
plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java
View File

@ -35,7 +35,6 @@
* </ul>
*/
/* List of renames that took place:
renamed: core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/AliasRoutingTests.java
renamed: core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/AvgTests.java
renamed: core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketScriptTests.java
renamed: core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketSelectorTests.java
@ -63,14 +62,12 @@
renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java
renamed: core/src/test/java/org/elasticsearch/search/aggregations/metrics/MinIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinTests.java
renamed: core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/PercolatorTests.java
renamed: core/src/test/java/org/elasticsearch/search/rescore/QueryRescorerIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/QueryRescorerTests.java
renamed: core/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java
renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RangeTests.java
renamed: core/src/test/java/org/elasticsearch/script/ScriptIndexSettingsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptIndexSettingsTests.java
renamed: core/src/test/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java
renamed: core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptedMetricTests.java
renamed: core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
renamed: core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchQueryTests.java
renamed: core/src/test/java/org/elasticsearch/search/stats/SearchStatsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java
renamed: core/src/test/java/org/elasticsearch/search/timeout/SearchTimeoutIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchTimeoutTests.java
renamed: core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SignificantTermsSignificanceScoreTests.java

31
plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchQueryTests.java → plugins/lang-groovy/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package org.elasticsearch.messy.tests;
package org.elasticsearch.search.query;
import org.apache.lucene.util.English;
import org.elasticsearch.Version;
@ -33,13 +33,11 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.query.*;
import org.elasticsearch.index.search.MatchQuery.Type;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.index.search.MatchQuery;
import org.elasticsearch.index.search.MatchQuery.Type;
import org.elasticsearch.indices.cache.query.terms.TermsLookup;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.groovy.GroovyPlugin;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilders;
@ -50,8 +48,6 @@ import org.joda.time.format.ISODateTimeFormat;
import org.junit.Test;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.Random;
import java.util.concurrent.ExecutionException;
@ -59,18 +55,12 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.*;
import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction;
import static org.elasticsearch.test.VersionUtils.randomVersion;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*;
import static org.hamcrest.Matchers.*;
public class SearchQueryTests extends ESIntegTestCase {
public class SearchQueryIT extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Collections.singleton(GroovyPlugin.class);
}
@Override
protected int maximumNumberOfShards() {
return 7;
@ -112,6 +102,7 @@ public class SearchQueryTests extends ESIntegTestCase {
assertThat(hits[0].score(), allOf(greaterThan(hits[1].getScore()), greaterThan(hits[2].getScore())));
}
@Test // see #3952
public void testEmptyQueryString() throws ExecutionException, InterruptedException, IOException {
createIndex("test");
@ -1945,14 +1936,14 @@ public class SearchQueryTests extends ESIntegTestCase {
public void testMinScore() throws ExecutionException, InterruptedException {
createIndex("test");
indexRandom(true,
client().prepareIndex("test", "test", "1").setSource("score", 1.5),
client().prepareIndex("test", "test", "2").setSource("score", 1.0),
client().prepareIndex("test", "test", "3").setSource("score", 2.0),
client().prepareIndex("test", "test", "4").setSource("score", 0.5));
client().prepareIndex("test", "test", "1").setSource("score", 1.5).get();
client().prepareIndex("test", "test", "2").setSource("score", 1.0).get();
client().prepareIndex("test", "test", "3").setSource("score", 2.0).get();
client().prepareIndex("test", "test", "4").setSource("score", 0.5).get();
refresh();
SearchResponse searchResponse = client().prepareSearch("test").setQuery(
functionScoreQuery(scriptFunction(new Script("_doc['score'].value")))).setMinScore(1.5f).get();
functionScoreQuery(ScoreFunctionBuilders.fieldValueFactorFunction("score").missing(1.0)).setMinScore(1.5f)).get();
assertHitCount(searchResponse, 2);
assertFirstHit(searchResponse, hasId("3"));
assertSecondHit(searchResponse, hasId("1"));

16
plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.script.javascript;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.Scorer;
import org.elasticsearch.SpecialPermission;
import org.elasticsearch.bootstrap.BootstrapInfo;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
@ -36,6 +37,10 @@ import org.mozilla.javascript.*;
import org.mozilla.javascript.Script;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.security.CodeSource;
import java.security.cert.Certificate;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
@ -105,7 +110,11 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements
try {
ctx.setWrapFactory(wrapFactory);
ctx.setOptimizationLevel(optimizationLevel);
return ctx.compileString(script, generateScriptName(), 1, null);
ctx.setSecurityController(new PolicySecurityController());
return ctx.compileString(script, generateScriptName(), 1,
new CodeSource(new URL("file:" + BootstrapInfo.UNTRUSTED_CODEBASE), (Certificate[]) null));
} catch (MalformedURLException e) {
throw new RuntimeException(e);
} finally {
Context.exit();
}
@ -189,11 +198,6 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements
}
}
@Override
public Object unwrap(Object value) {
return ScriptValueConverter.unwrapValue(value);
}
private String generateScriptName() {
return "Script" + counter.incrementAndGet() + ".js";
}

5
plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineTests.java
View File

@ -97,9 +97,10 @@ public class JavaScriptScriptEngineTests extends ESTestCase {
ctx.put("obj1", obj1);
vars.put("ctx", ctx);
se.execute(new CompiledScript(ScriptService.ScriptType.INLINE, "testJavaScriptObjectMapInter", "js",
ExecutableScript executable = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testJavaScriptObjectMapInter", "js",
se.compile("ctx.obj2 = {}; ctx.obj2.prop2 = 'value2'; ctx.obj1.prop1 = 'uvalue1'")), vars);
ctx = (Map<String, Object>) se.unwrap(vars.get("ctx"));
executable.run();
ctx = (Map<String, Object>) executable.unwrap(vars.get("ctx"));
assertThat(ctx.containsKey("obj1"), equalTo(true));
assertThat((String) ((Map<String, Object>) ctx.get("obj1")).get("prop1"), equalTo("uvalue1"));
assertThat(ctx.containsKey("obj2"), equalTo(true));

89
plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptSecurityTests.java Normal file
View File

@ -0,0 +1,89 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.javascript;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.test.ESTestCase;
import org.junit.After;
import org.junit.Before;
import org.mozilla.javascript.WrappedException;
import java.util.HashMap;
import java.util.Map;
/**
* Tests for the Javascript security permissions
*/
public class JavaScriptSecurityTests extends ESTestCase {
private JavaScriptScriptEngineService se;
@Before
public void setup() {
se = new JavaScriptScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
}
@After
public void close() {
se.close();
}
/** runs a script */
private void doTest(String script) {
Map<String, Object> vars = new HashMap<String, Object>();
se.execute(new CompiledScript(ScriptService.ScriptType.INLINE, "test", "js", se.compile(script)), vars);
}
/** asserts that a script runs without exception */
private void assertSuccess(String script) {
doTest(script);
}
/** assert that a security exception is hit */
private void assertFailure(String script) {
try {
doTest(script);
fail("did not get expected exception");
} catch (WrappedException expected) {
Throwable cause = expected.getCause();
assertNotNull(cause);
assertTrue("unexpected exception: " + cause, cause instanceof SecurityException);
}
}
/** Test some javascripts that are ok */
public void testOK() {
assertSuccess("1 + 2");
assertSuccess("Math.cos(Math.PI)");
}
/** Test some javascripts that should hit security exception */
public void testNotOK() {
// sanity check :)
assertFailure("java.lang.Runtime.getRuntime().halt(0)");
// check a few things more restrictive than the ordinary policy
// no network
assertFailure("new java.net.Socket(\"localhost\", 1024)");
// no files
assertFailure("java.io.File.createTempFile(\"test\", \"tmp\")");
}
}

38
plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
View File

@ -20,8 +20,11 @@
package org.elasticsearch.script.python;
import java.io.IOException;
import java.security.AccessControlContext;
import java.security.AccessController;
import java.security.Permissions;
import java.security.PrivilegedAction;
import java.security.ProtectionDomain;
import java.util.Map;
import org.apache.lucene.index.LeafReaderContext;
@ -125,18 +128,14 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri
public Object execute(CompiledScript compiledScript, Map<String, Object> vars) {
PyObject pyVars = Py.java2py(vars);
interp.setLocals(pyVars);
PyObject ret = interp.eval((PyCode) compiledScript.compiled());
// eval the script with reduced privileges
PyObject ret = evalRestricted((PyCode) compiledScript.compiled());
if (ret == null) {
return null;
}
return ret.__tojava__(Object.class);
}
@Override
public Object unwrap(Object value) {
return unwrapValue(value);
}
@Override
public void close() {
interp.cleanup();
@ -171,7 +170,8 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri
@Override
public Object run() {
interp.setLocals(pyVars);
PyObject ret = interp.eval(code);
// eval the script with reduced privileges
PyObject ret = evalRestricted(code);
if (ret == null) {
return null;
}
@ -229,7 +229,8 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri
@Override
public Object run() {
interp.setLocals(pyVars);
PyObject ret = interp.eval(code);
// eval the script with reduced privileges
PyObject ret = evalRestricted(code);
if (ret == null) {
return null;
}
@ -257,6 +258,27 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri
}
}
// we don't have a way to specify codesource for generated jython classes,
// so we just run them with a special context to reduce privileges
private static final AccessControlContext PY_CONTEXT;
static {
Permissions none = new Permissions();
none.setReadOnly();
PY_CONTEXT = new AccessControlContext(new ProtectionDomain[] {
new ProtectionDomain(null, none)
});
}
/** Evaluates with reduced privileges */
private final PyObject evalRestricted(final PyCode code) {
// eval the script with reduced privileges
return AccessController.doPrivileged(new PrivilegedAction<PyObject>() {
@Override
public PyObject run() {
return interp.eval(code);
}
}, PY_CONTEXT);
}
public static Object unwrapValue(Object value) {
if (value == null) {

5
plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java
View File

@ -89,9 +89,10 @@ public class PythonScriptEngineTests extends ESTestCase {
ctx.put("obj1", obj1);
vars.put("ctx", ctx);
se.execute(new CompiledScript(ScriptService.ScriptType.INLINE, "testObjectInterMap", "python",
ExecutableScript executable = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testObjectInterMap", "python",
se.compile("ctx['obj2'] = { 'prop2' : 'value2' }; ctx['obj1']['prop1'] = 'uvalue1'")), vars);
ctx = (Map<String, Object>) se.unwrap(vars.get("ctx"));
executable.run();
ctx = (Map<String, Object>) executable.unwrap(vars.get("ctx"));
assertThat(ctx.containsKey("obj1"), equalTo(true));
assertThat((String) ((Map<String, Object>) ctx.get("obj1")).get("prop1"), equalTo("uvalue1"));
assertThat(ctx.containsKey("obj2"), equalTo(true));

92
plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonSecurityTests.java Normal file
View File

@ -0,0 +1,92 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script.python;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.test.ESTestCase;
import org.junit.After;
import org.junit.Before;
import org.python.core.PyException;
import java.util.HashMap;
import java.util.Map;
/**
* Tests for Python security permissions
*/
public class PythonSecurityTests extends ESTestCase {
private PythonScriptEngineService se;
@Before
public void setup() {
se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
}
@After
public void close() {
// We need to clear some system properties
System.clearProperty("python.cachedir.skip");
System.clearProperty("python.console.encoding");
se.close();
}
/** runs a script */
private void doTest(String script) {
Map<String, Object> vars = new HashMap<String, Object>();
se.execute(new CompiledScript(ScriptService.ScriptType.INLINE, "test", "python", se.compile(script)), vars);
}
/** asserts that a script runs without exception */
private void assertSuccess(String script) {
doTest(script);
}
/** assert that a security exception is hit */
private void assertFailure(String script) {
try {
doTest(script);
fail("did not get expected exception");
} catch (PyException expected) {
Throwable cause = expected.getCause();
assertNotNull("null cause for exception: " + expected, cause);
assertTrue("unexpected exception: " + cause, cause instanceof SecurityException);
}
}
/** Test some py scripts that are ok */
public void testOK() {
assertSuccess("1 + 2");
assertSuccess("from java.lang import Math\nMath.cos(0)");
}
/** Test some py scripts that should hit security exception */
public void testNotOK() {
// sanity check :)
assertFailure("from java.lang import Runtime\nRuntime.getRuntime().halt(0)");
// check a few things more restrictive than the ordinary policy
// no network
assertFailure("from java.net import Socket\nSocket(\"localhost\", 1024)");
// no files
assertFailure("from java.io import File\nFile.createTempFile(\"test\", \"tmp\")");
}
}

2
rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
View File

@ -7,7 +7,7 @@
"paths": ["/_cluster/health", "/_cluster/health/{index}"],
"parts": {
"index": {
"type" : "string",
"type" : "list",
"description" : "Limit the information returned to a specific index"
}
},

3
rest-api-spec/src/main/resources/rest-api-spec/api/delete_template.json
View File

@ -8,7 +8,8 @@
"parts": {
"id": {
"type" : "string",
"description" : "Template ID"
"description" : "Template ID",
"required" : true
}
},
"params" : {

4
rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
View File

@ -7,9 +7,9 @@
"paths": ["/{index}/_close"],
"parts": {
"index": {
"type" : "string",
"type" : "list",
"required" : true,
"description" : "The name of the index"
"description" : "A comma separated list of indices to close"
}
},
"params": {

6
rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
View File

@ -3,8 +3,8 @@
"documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-field-mapping.html",
"methods": ["GET"],
"url": {
"path": "/_mapping/field/{field}",
"paths": ["/_mapping/field/{field}", "/{index}/_mapping/field/{field}", "/_mapping/{type}/field/{field}", "/{index}/_mapping/{type}/field/{field}"],
"path": "/_mapping/field/{fields}",
"paths": ["/_mapping/field/{fields}", "/{index}/_mapping/field/{fields}", "/_mapping/{type}/field/{fields}", "/{index}/_mapping/{type}/field/{fields}"],
"parts": {
"index": {
"type" : "list",
@ -14,7 +14,7 @@
"type" : "list",
"description" : "A comma-separated list of document types"
},
"field": {
"fields": {
"type" : "list",
"description" : "A comma-separated list of fields",
"required" : true

4
rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json
View File

@ -10,9 +10,9 @@
],
"parts": {
"name": {
"type": "string",
"type": "list",
"required": false,
"description": "The name of the template"
"description": "The comma separated names of the index templates"
}
},
"params": {

4
rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
View File

@ -7,9 +7,9 @@
"paths": ["/{index}/_open"],
"parts": {
"index": {
"type" : "string",
"type" : "list",
"required" : true,
"description" : "The name of the index"
"description" : "A comma separated list of indices to open"
}
},
"params": {

2
rest-api-spec/src/main/resources/rest-api-spec/test/search/80_date_math_index_names.yaml
View File

@ -2,6 +2,6 @@
"Missing index with catch":
- do:
catch: /index=logstash-\d{4}\.\d{2}\.\d{2}/
catch: /logstash-\d{4}\.\d{2}\.\d{2}/
search:
index: <logstash-{now/M}>