honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into feature/client_aggs_parsing

This commit is contained in:
Tanguy Leroux 2017-05-19 13:13:00 +02:00
parent 4c34ea8fc8 55af1f7a2b
commit 83aa00b3f6
70 changed files with 903 additions and 1557 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
TESTING.asciidoc
View File

@ -449,6 +449,26 @@ Note: Starting vagrant VM outside of the elasticsearch folder requires to
indicates the folder that contains the Vagrantfile using the VAGRANT_CWD
environment variable.
== Testing backwards compatibility
Backwards compatibility tests exist to test upgrading from each supported version
to the current version. To run all backcompat tests use:
-------------------------------------------------
gradle bwcTest
-------------------------------------------------
A specific version can be tested as well. For example, to test backcompat with
version 5.3.2 run:
-------------------------------------------------
gradle v5.3.2#bwcTest
-------------------------------------------------
When running `gradle check`, some minimal backcompat checks are run. Which version
is tested depends on the branch. On master, this will test against the current
stable branch. On the stable branch, it will test against the latest release
branch. Finally, on a release branch, it will test against the most recent release.
== Coverage analysis

34
build.gradle
View File

@ -24,6 +24,7 @@ import org.eclipse.jgit.lib.RepositoryBuilder
import org.gradle.plugins.ide.eclipse.model.SourceFolder
import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.Version
// common maven publishing configuration
subprojects {
@ -62,11 +63,11 @@ configure(subprojects.findAll { it.projectDir.toPath().startsWith(rootPath) }) {
}
// introspect all versions of ES that may be tested agains for backwards compatibility
String currentVersion = VersionProperties.elasticsearch.minus('-SNAPSHOT')
int prevMajor = Integer.parseInt(currentVersion.split('\\.')[0]) - 1
Version currentVersion = Version.fromString(VersionProperties.elasticsearch.minus('-SNAPSHOT'))
int prevMajor = currentVersion.major - 1
File versionFile = file('core/src/main/java/org/elasticsearch/Version.java')
List<String> versionLines = versionFile.readLines('UTF-8')
List<String> versions = []
List<Version> versions = []
// keep track of the previous major version's last minor, so we know where wire compat begins
int prevMinorIndex = -1 // index in the versions list of the last minor from the prev major
int lastPrevMinor = -1 // the minor version number from the prev major we most recently seen
@ -76,9 +77,10 @@ for (String line : versionLines) {
int major = Integer.parseInt(match.group(1))
int minor = Integer.parseInt(match.group(2))
int bugfix = Integer.parseInt(match.group(3))
String versionStr = "${major}.${minor}.${bugfix}"
if (currentVersion != versionStr) {
versions.add(versionStr)
boolean unreleased = match.group(4) != null
Version foundVersion = new Version(major, minor, bugfix, false, unreleased)
if (currentVersion != foundVersion) {
versions.add(foundVersion)
}
if (major == prevMajor && minor > lastPrevMinor) {
prevMinorIndex = versions.size() - 1
@ -86,16 +88,18 @@ for (String line : versionLines) {
}
}
}
if (versions.toSorted() != versions) {
throw new GradleException('Versions.java contains out of order version constants')
if (versions.toSorted { it.id } != versions) {
println "Versions: ${versions}"
throw new GradleException("Versions.java contains out of order version constants")
}
if (currentVersion.split('\\.')[2].split('-')[0] == '0') {
if (currentVersion.bugfix == 0) {
// If on a release branch, after the initial release of that branch, the bugfix version will
// be bumped, and will be != 0. On master and N.x branches, we want to test against the
// unreleased version of closest branch. So for those cases, the version includes -SNAPSHOT,
// and the bwc-zip distribution will checkout and build that version. The version parsing
// logic above pulls the bugfix version, and then strips off any prerelease version
versions[-1] += '-SNAPSHOT'
// and the bwc-zip distribution will checkout and build that version.
Version last = versions[-1]
versions[-1] = new Version(last.major, last.minor, last.bugfix,
true, last.unreleased)
}
// injecting groovy property variables into all projects
@ -154,7 +158,6 @@ subprojects {
"org.elasticsearch.client:transport:${version}": ':client:transport',
"org.elasticsearch.test:framework:${version}": ':test:framework',
"org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:integ-test-zip',
"org.elasticsearch.distribution.zip:elasticsearch:${wireCompatVersions[-1]}": ':distribution:bwc-zip',
"org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip',
"org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar',
"org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:rpm',
@ -167,6 +170,11 @@ subprojects {
"org.elasticsearch.plugin:parent-join-client:${version}": ':modules:parent-join',
"org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
]
if (wireCompatVersions[-1].snapshot) {
// if the most previous version is a snapshot, we need to connect that version to the
// bwc-zip project which will checkout and build that snapshot version
ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc-zip'
}
project.afterEvaluate {
configurations.all {
resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->

85
buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy Normal file
View File

@ -0,0 +1,85 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle
/**
* Encapsulates comparison and printing logic for an x.y.z version.
*/
public class Version {
final int major
final int minor
final int bugfix
final int id
final boolean snapshot
/**
* Is the vesion listed as {@code _UNRELEASED} in Version.java.
*/
final boolean unreleased
public Version(int major, int minor, int bugfix, boolean snapshot,
boolean unreleased) {
this.major = major
this.minor = minor
this.bugfix = bugfix
this.snapshot = snapshot
this.id = major * 100000 + minor * 1000 + bugfix * 10 +
(snapshot ? 1 : 0)
this.unreleased = unreleased
}
public static Version fromString(String s) {
String[] parts = s.split('\\.')
String bugfix = parts[2]
boolean snapshot = false
if (bugfix.contains('-')) {
snapshot = bugfix.endsWith('-SNAPSHOT')
bugfix = bugfix.split('-')[0]
}
return new Version(parts[0] as int, parts[1] as int, bugfix as int,
snapshot, false)
}
@Override
public String toString() {
String snapshotStr = snapshot ? '-SNAPSHOT' : ''
return "${major}.${minor}.${bugfix}${snapshotStr}"
}
public boolean equals(Version compareTo) {
return id == compareTo.id
}
public boolean before(String compareTo) {
return id < fromString(compareTo).id
}
public boolean onOrBefore(String compareTo) {
return id <= fromString(compareTo).id
}
public boolean onOrAfter(String compareTo) {
return id >= fromString(compareTo).id
}
public boolean after(String compareTo) {
return id > fromString(compareTo).id
}
}

52
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
View File

@ -72,25 +72,25 @@ class ClusterFormationTasks {
throw new GradleException("bwcVersion must not be null if numBwcNodes is > 0")
}
// this is our current version distribution configuration we use for all kinds of REST tests etc.
String distroConfigName = "${prefix}_elasticsearchDistro"
Configuration currentDistro = project.configurations.create(distroConfigName)
Configuration currentDistro = project.configurations.create("${prefix}_elasticsearchDistro")
Configuration bwcDistro = project.configurations.create("${prefix}_elasticsearchBwcDistro")
Configuration bwcPlugins = project.configurations.create("${prefix}_elasticsearchBwcPlugins")
configureDistributionDependency(project, config.distribution, currentDistro, VersionProperties.elasticsearch)
if (config.bwcVersion != null && config.numBwcNodes > 0) {
if (config.numBwcNodes > 0) {
if (config.bwcVersion == null) {
throw new IllegalArgumentException("Must specify bwcVersion when numBwcNodes > 0")
}
// if we have a cluster that has a BWC cluster we also need to configure a dependency on the BWC version
// this version uses the same distribution etc. and only differs in the version we depend on.
// from here on everything else works the same as if it's the current version, we fetch the BWC version
// from mirrors using gradles built-in mechanism etc.
project.configurations {
elasticsearchBwcDistro
elasticsearchBwcPlugins
}
configureDistributionDependency(project, config.distribution, project.configurations.elasticsearchBwcDistro, config.bwcVersion)
configureDistributionDependency(project, config.distribution, bwcDistro, config.bwcVersion)
for (Map.Entry<String, Project> entry : config.plugins.entrySet()) {
configureBwcPluginDependency("${prefix}_elasticsearchBwcPlugins", project, entry.getValue(),
project.configurations.elasticsearchBwcPlugins, config.bwcVersion)
configureBwcPluginDependency("${prefix}_elasticsearchBwcPlugins", project, entry.getValue(), bwcPlugins, config.bwcVersion)
}
project.configurations.elasticsearchBwcDistro.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
project.configurations.elasticsearchBwcPlugins.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
bwcDistro.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
bwcPlugins.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
}
for (int i = 0; i < config.numNodes; i++) {
// we start N nodes and out of these N nodes there might be M bwc nodes.
@ -99,7 +99,7 @@ class ClusterFormationTasks {
Configuration distro = currentDistro
if (i < config.numBwcNodes) {
elasticsearchVersion = config.bwcVersion
distro = project.configurations.elasticsearchBwcDistro
distro = bwcDistro
}
NodeInfo node = new NodeInfo(config, i, project, prefix, elasticsearchVersion, sharedDir)
nodes.add(node)
@ -169,9 +169,9 @@ class ClusterFormationTasks {
if (node.config.plugins.isEmpty() == false) {
if (node.nodeVersion == VersionProperties.elasticsearch) {
setup = configureCopyPluginsTask(taskName(prefix, node, 'copyPlugins'), project, setup, node)
setup = configureCopyPluginsTask(taskName(prefix, node, 'copyPlugins'), project, setup, node, prefix)
} else {
setup = configureCopyBwcPluginsTask(taskName(prefix, node, 'copyBwcPlugins'), project, setup, node)
setup = configureCopyBwcPluginsTask(taskName(prefix, node, 'copyBwcPlugins'), project, setup, node, prefix)
}
}
@ -184,7 +184,7 @@ class ClusterFormationTasks {
// install plugins
for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
String actionName = pluginTaskName('install', plugin.getKey(), 'Plugin')
setup = configureInstallPluginTask(taskName(prefix, node, actionName), project, setup, node, plugin.getValue())
setup = configureInstallPluginTask(taskName(prefix, node, actionName), project, setup, node, plugin.getValue(), prefix)
}
// sets up any extra config files that need to be copied over to the ES instance;
@ -379,7 +379,7 @@ class ClusterFormationTasks {
* For each plugin, if the plugin has rest spec apis in its tests, those api files are also copied
* to the test resources for this project.
*/
static Task configureCopyPluginsTask(String name, Project project, Task setup, NodeInfo node) {
static Task configureCopyPluginsTask(String name, Project project, Task setup, NodeInfo node, String prefix) {
Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup)
List<FileCollection> pluginFiles = []
@ -387,7 +387,7 @@ class ClusterFormationTasks {
Project pluginProject = plugin.getValue()
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
String configurationName = "_plugin_${pluginProject.path}"
String configurationName = "_plugin_${prefix}_${pluginProject.path}"
Configuration configuration = project.configurations.findByName(configurationName)
if (configuration == null) {
configuration = project.configurations.create(configurationName)
@ -417,25 +417,27 @@ class ClusterFormationTasks {
}
/** Configures task to copy a plugin based on a zip file resolved using dependencies for an older version */
static Task configureCopyBwcPluginsTask(String name, Project project, Task setup, NodeInfo node) {
static Task configureCopyBwcPluginsTask(String name, Project project, Task setup, NodeInfo node, String prefix) {
Configuration bwcPlugins = project.configurations.getByName("${prefix}_elasticsearchBwcPlugins")
for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
Project pluginProject = plugin.getValue()
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
String configurationName = "_plugin_bwc_${pluginProject.path}"
String configurationName = "_plugin_bwc_${prefix}_${pluginProject.path}"
Configuration configuration = project.configurations.findByName(configurationName)
if (configuration == null) {
configuration = project.configurations.create(configurationName)
}
final String depName = pluginProject.extensions.findByName('esplugin').name
Dependency dep = project.configurations.elasticsearchBwcPlugins.dependencies.find {
Dependency dep = bwcPlugins.dependencies.find {
it.name == depName
}
configuration.dependencies.add(dep)
}
Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup) {
from project.configurations.elasticsearchBwcPlugins
from bwcPlugins
into node.pluginsTmpDir
}
return copyPlugins
@ -455,12 +457,12 @@ class ClusterFormationTasks {
return installModule
}
static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Project plugin) {
static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Project plugin, String prefix) {
final FileCollection pluginZip;
if (node.nodeVersion != VersionProperties.elasticsearch) {
pluginZip = project.configurations.getByName("_plugin_bwc_${plugin.path}")
pluginZip = project.configurations.getByName("_plugin_bwc_${prefix}_${plugin.path}")
} else {
pluginZip = project.configurations.getByName("_plugin_${plugin.path}")
pluginZip = project.configurations.getByName("_plugin_${prefix}_${plugin.path}")
}
// delay reading the file location until execution time by wrapping in a closure within a GString
Object file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}"

2
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
View File

@ -51,8 +51,6 @@ public class RestIntegTestTask extends DefaultTask {
boolean includePackaged = false
public RestIntegTestTask() {
description = 'Runs rest tests against an elasticsearch cluster.'
group = JavaBasePlugin.VERIFICATION_GROUP
runner = project.tasks.create("${name}Runner", RandomizedTestingTask.class)
super.dependsOn(runner)
clusterInit = project.tasks.create(name: "${name}Cluster#init", dependsOn: project.testClasses)

3
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy
View File

@ -22,6 +22,7 @@ import org.elasticsearch.gradle.BuildPlugin
import org.gradle.api.InvalidUserDataException
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.plugins.JavaBasePlugin
/**
* Adds support for starting an Elasticsearch cluster before running integration
@ -43,6 +44,8 @@ public class RestTestPlugin implements Plugin<Project> {
}
RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
integTest.description = 'Runs rest tests against an elasticsearch cluster.'
integTest.group = JavaBasePlugin.VERIFICATION_GROUP
integTest.clusterConfig.distribution = 'zip' // rest tests should run with the real zip
integTest.mustRunAfter(project.precommit)
project.check.dependsOn(integTest)

55
buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
View File

@ -82,29 +82,6 @@ class VagrantTestPlugin implements Plugin<Project> {
}
}
private static Set<String> listVersions(Project project) {
Node xml
new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
xml = new XmlParser().parse(s)
}
Set<String> versions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /[5]\.\d\.\d/ })
if (versions.isEmpty() == false) {
return versions;
}
// If no version is found, we run the tests with the current version
return Collections.singleton(project.version);
}
private static File getVersionsFile(Project project) {
File versions = new File(project.projectDir, 'versions');
if (versions.exists() == false) {
// Use the elasticsearch's versions file from project :qa:vagrant
versions = project.project(":qa:vagrant").file('versions')
}
return versions
}
private static void configureBatsRepositories(Project project) {
RepositoryHandler repos = project.repositories
@ -140,8 +117,7 @@ class VagrantTestPlugin implements Plugin<Project> {
String upgradeFromVersion = System.getProperty("tests.packaging.upgradeVersion");
if (upgradeFromVersion == null) {
List<String> availableVersions = getVersionsFile(project).readLines('UTF-8')
upgradeFromVersion = availableVersions[new Random(seed).nextInt(availableVersions.size())]
upgradeFromVersion = project.indexCompatVersions[new Random(seed).nextInt(project.indexCompatVersions.size())]
}
DISTRIBUTION_ARCHIVES.each {
@ -186,7 +162,6 @@ class VagrantTestPlugin implements Plugin<Project> {
Task createBatsDirsTask = project.tasks.create('createBatsDirs')
createBatsDirsTask.outputs.dir batsDir
createBatsDirsTask.dependsOn project.tasks.vagrantVerifyVersions
createBatsDirsTask.doLast {
batsDir.mkdirs()
}
@ -252,32 +227,6 @@ class VagrantTestPlugin implements Plugin<Project> {
vagrantSetUpTask.dependsOn copyBatsTests, copyBatsUtils, copyBatsArchives, createVersionFile, createUpgradeFromFile
}
private static void createUpdateVersionsTask(Project project) {
project.tasks.create('vagrantUpdateVersions') {
description 'Update file containing options for the\n "starting" version in the "upgrade from" packaging tests.'
group 'Verification'
doLast {
File versions = getVersionsFile(project)
versions.setText(listVersions(project).join('\n') + '\n', 'UTF-8')
}
}
}
private static void createVerifyVersionsTask(Project project) {
project.tasks.create('vagrantVerifyVersions') {
description 'Update file containing options for the\n "starting" version in the "upgrade from" packaging tests.'
group 'Verification'
doLast {
Set<String> versions = listVersions(project)
Set<String> actualVersions = new TreeSet<>(getVersionsFile(project).readLines('UTF-8'))
if (!versions.equals(actualVersions)) {
throw new GradleException("out-of-date versions " + actualVersions +
", expected " + versions + "; run gradle vagrantUpdateVersions")
}
}
}
}
private static void createCheckVagrantVersionTask(Project project) {
project.tasks.create('vagrantCheckVersion', Exec) {
description 'Check the Vagrant version'
@ -342,8 +291,6 @@ class VagrantTestPlugin implements Plugin<Project> {
createCleanTask(project)
createStopTask(project)
createSmokeTestTask(project)
createUpdateVersionsTask(project)
createVerifyVersionsTask(project)
createCheckVagrantVersionTask(project)
createCheckVirtualBoxVersionTask(project)
createPrepareVagrantTestEnvTask(project)

11
client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java
View File

@ -139,11 +139,12 @@ final class RequestLogger {
* Creates curl output for given response
*/
static String buildTraceResponse(HttpResponse httpResponse) throws IOException {
String responseLine = "# " + httpResponse.getStatusLine().toString();
StringBuilder responseLine = new StringBuilder();
responseLine.append("# ").append(httpResponse.getStatusLine());
for (Header header : httpResponse.getAllHeaders()) {
responseLine += "\n# " + header.getName() + ": " + header.getValue();
responseLine.append("\n# ").append(header.getName()).append(": ").append(header.getValue());
}
responseLine += "\n#";
responseLine.append("\n#");
HttpEntity entity = httpResponse.getEntity();
if (entity != null) {
if (entity.isRepeatable() == false) {
@ -158,11 +159,11 @@ final class RequestLogger {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(entity.getContent(), charset))) {
String line;
while( (line = reader.readLine()) != null) {
responseLine += "\n# " + line;
responseLine.append("\n# ").append(line);
}
}
}
return responseLine;
return responseLine.toString();
}
private static String getUri(RequestLine requestLine) {

40
core/build.gradle
View File

@ -277,3 +277,43 @@ if (isEclipse == false || project.path == ":core-tests") {
check.dependsOn integTest
integTest.mustRunAfter test
}
task('verifyVersions') {
description 'Verifies that all released versions that are indexed compatible are listed in Version.java.'
group 'Verification'
enabled = false == gradle.startParameter.isOffline()
doLast {
// Read the list from maven central
Node xml
new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
xml = new XmlParser().parse(s)
}
Set<String> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ })
// Limit the known versions to those that should be wire compatible
String currentVersion = versions.elasticsearch.minus('-SNAPSHOT')
int prevMajor = Integer.parseInt(currentVersion.split('\\.')[0]) - 1
if (prevMajor == 4) {
// 4 didn't exist, it was 2.
prevMajor = 2;
}
knownVersions = knownVersions.findAll { Integer.parseInt(it.split('\\.')[0]) >= prevMajor }
/* Limit the listed versions to those that have been marked as released.
* Versions not marked as released don't get the same testing and we want
* to make sure that we flip all unreleased versions to released as soon
* as possible after release. */
Set<String> actualVersions = new TreeSet<>(
indexCompatVersions
.findAll { false == it.unreleased }
.collect { it.toString() })
// Finally, compare!
if (!knownVersions.equals(actualVersions)) {
throw new GradleException("out-of-date versions\nActual :" +
actualVersions + "\nExpected:" + knownVersions +
"; update Version.java")
}
}
}
check.dependsOn(verifyVersions)

9
core/src/main/java/org/elasticsearch/Version.java
View File

@ -278,9 +278,12 @@ public class Version implements Comparable<Version> {
public Version minimumCompatibilityVersion() {
final int bwcMajor;
final int bwcMinor;
if (this.onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
bwcMajor = major - 1;
bwcMinor = 4;
if (major == 6) { // we only specialize for current major here
bwcMajor = Version.V_5_4_0.major;
bwcMinor = Version.V_5_4_0.minor;
} else if (major > 6) { // all the future versions are compatible with first minor...
bwcMajor = major -1;
bwcMinor = 0;
} else {
bwcMajor = major;
bwcMinor = 0;

20
core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
View File

@ -23,7 +23,6 @@ import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.logging.log4j.util.Supplier;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.Version;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.delete.DeleteRequest;
@ -31,8 +30,8 @@ import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.replication.ReplicationOperation;
import org.elasticsearch.action.support.TransportActions;
import org.elasticsearch.action.support.replication.ReplicationOperation;
import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo;
import org.elasticsearch.action.support.replication.TransportWriteAction;
import org.elasticsearch.action.update.UpdateHelper;
@ -56,7 +55,6 @@ import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.engine.VersionConflictEngineException;
import org.elasticsearch.index.get.GetResult;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.Mapping;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.index.seqno.SequenceNumbersService;
@ -414,13 +412,6 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
FAILURE
}
static {
assert Version.CURRENT.minimumCompatibilityVersion().after(Version.V_6_0_0_alpha1_UNRELEASED) == false:
"Remove logic handling NoOp result from primary response; see TODO in replicaItemExecutionMode" +
" as the current minimum compatible version [" +
Version.CURRENT.minimumCompatibilityVersion() + "] is after 6.0";
}
/**
* Determines whether a bulk item request should be executed on the replica.
* @return {@link ReplicaItemExecutionMode#NORMAL} upon normal primary execution with no failures
@ -436,10 +427,11 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
? ReplicaItemExecutionMode.FAILURE // we have a seq no generated with the failure, replicate as no-op
: ReplicaItemExecutionMode.NOOP; // no seq no generated, ignore replication
} else {
// NOTE: write requests originating from pre-6.0 nodes can send a no-op operation to
// the replica; we ignore replication
// TODO: remove noOp result check from primary response, when pre-6.0 nodes are not supported
// we should return ReplicationItemExecutionMode.NORMAL instead
// TODO: once we know for sure that every operation that has been processed on the primary is assigned a seq#
// (i.e., all nodes on the cluster are on v6.0.0 or higher) we can use the existence of a seq# to indicate whether
// an operation should be processed or be treated as a noop. This means we could remove this method and the
// ReplicaItemExecutionMode enum and have a simple boolean check for seq != UNASSIGNED_SEQ_NO which will work for
// both failures and indexing operations.
return primaryResponse.getResponse().getResult() != DocWriteResponse.Result.NOOP
? ReplicaItemExecutionMode.NORMAL // execution successful on primary
: ReplicaItemExecutionMode.NOOP; // ignore replication

7
core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
View File

@ -19,8 +19,6 @@
package org.elasticsearch.common.settings;
import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction;
import org.elasticsearch.transport.RemoteClusterService;
import org.elasticsearch.transport.RemoteClusterAware;
import org.elasticsearch.action.search.TransportSearchAction;
import org.elasticsearch.action.support.AutoCreateIndex;
import org.elasticsearch.action.support.DestructiveOperations;
@ -88,6 +86,8 @@ import org.elasticsearch.search.SearchModule;
import org.elasticsearch.search.SearchService;
import org.elasticsearch.search.fetch.subphase.highlight.FastVectorHighlighter;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.RemoteClusterAware;
import org.elasticsearch.transport.RemoteClusterService;
import org.elasticsearch.transport.TcpTransport;
import org.elasticsearch.transport.Transport;
import org.elasticsearch.transport.TransportService;
@ -304,6 +304,8 @@ public final class ClusterSettings extends AbstractScopedSettings {
ScriptService.SCRIPT_CACHE_EXPIRE_SETTING,
ScriptService.SCRIPT_MAX_SIZE_IN_BYTES,
ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE,
ScriptService.TYPES_ALLOWED_SETTING,
ScriptService.CONTEXTS_ALLOWED_SETTING,
IndicesService.INDICES_CACHE_CLEAN_INTERVAL_SETTING,
IndicesFieldDataCache.INDICES_FIELDDATA_CACHE_SIZE_KEY,
IndicesRequestCache.INDICES_CACHE_QUERY_SIZE,
@ -339,6 +341,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
ZenDiscovery.SEND_LEAVE_REQUEST_SETTING,
ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING,
ZenDiscovery.MASTER_ELECTION_IGNORE_NON_MASTER_PINGS_SETTING,
ZenDiscovery.MAX_PENDING_CLUSTER_STATES_SETTING,
UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING,
UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING,
UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_HOSTS_RESOLVE_TIMEOUT,

93
core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java
View File

@ -23,8 +23,8 @@ import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.Diff;
import org.elasticsearch.cluster.IncompatibleClusterStateVersionException;
@ -60,61 +60,53 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Supplier;
public class PublishClusterStateAction extends AbstractComponent {
public static final String SEND_ACTION_NAME = "internal:discovery/zen/publish/send";
public static final String COMMIT_ACTION_NAME = "internal:discovery/zen/publish/commit";
public static final String SETTINGS_MAX_PENDING_CLUSTER_STATES = "discovery.zen.publish.max_pending_cluster_states";
public interface IncomingClusterStateListener {
public interface NewPendingClusterStateListener {
/**
* called when a new incoming cluster state has been received.
* Should validate the incoming state and throw an exception if it's not a valid successor state.
*/
void onIncomingClusterState(ClusterState incomingState);
/** a new cluster state has been committed and is ready to process via {@link #pendingStatesQueue()} */
void onNewClusterState(String reason);
/**
* called when a cluster state has been committed and is ready to be processed
*/
void onClusterStateCommitted(String stateUUID, ActionListener<Void> processedListener);
}
private final TransportService transportService;
private final NamedWriteableRegistry namedWriteableRegistry;
private final Supplier<ClusterState> clusterStateSupplier;
private final NewPendingClusterStateListener newPendingClusterStatelistener;
private final IncomingClusterStateListener incomingClusterStateListener;
private final DiscoverySettings discoverySettings;
private final ClusterName clusterName;
private final PendingClusterStatesQueue pendingStatesQueue;
public PublishClusterStateAction(
Settings settings,
TransportService transportService,
NamedWriteableRegistry namedWriteableRegistry,
Supplier<ClusterState> clusterStateSupplier,
NewPendingClusterStateListener listener,
DiscoverySettings discoverySettings,
ClusterName clusterName) {
IncomingClusterStateListener incomingClusterStateListener,
DiscoverySettings discoverySettings) {
super(settings);
this.transportService = transportService;
this.namedWriteableRegistry = namedWriteableRegistry;
this.clusterStateSupplier = clusterStateSupplier;
this.newPendingClusterStatelistener = listener;
this.incomingClusterStateListener = incomingClusterStateListener;
this.discoverySettings = discoverySettings;
this.clusterName = clusterName;
this.pendingStatesQueue = new PendingClusterStatesQueue(logger, settings.getAsInt(SETTINGS_MAX_PENDING_CLUSTER_STATES, 25));
transportService.registerRequestHandler(SEND_ACTION_NAME, BytesTransportRequest::new, ThreadPool.Names.SAME, false, false,
new SendClusterStateRequestHandler());
transportService.registerRequestHandler(COMMIT_ACTION_NAME, CommitClusterStateRequest::new, ThreadPool.Names.SAME, false, false,
new CommitClusterStateRequestHandler());
}
public PendingClusterStatesQueue pendingStatesQueue() {
return pendingStatesQueue;
}
/**
* publishes a cluster change event to other nodes. if at least minMasterNodes acknowledge the change it is committed and will
* be processed by the master and the other nodes.
@ -387,7 +379,7 @@ public class PublishClusterStateAction extends AbstractComponent {
final ClusterState incomingState;
// If true we received full cluster state - otherwise diffs
if (in.readBoolean()) {
incomingState = ClusterState.readFrom(in, clusterStateSupplier.get().nodes().getLocalNode());
incomingState = ClusterState.readFrom(in, transportService.getLocalNode());
logger.debug("received full cluster state version [{}] with size [{}]", incomingState.version(),
request.bytes().length());
} else if (lastSeenClusterState != null) {
@ -399,10 +391,7 @@ public class PublishClusterStateAction extends AbstractComponent {
logger.debug("received diff for but don't have any local cluster state - requesting full state");
throw new IncompatibleClusterStateVersionException("have no local cluster state");
}
// sanity check incoming state
validateIncomingState(incomingState, lastSeenClusterState);
pendingStatesQueue.addPending(incomingState);
incomingClusterStateListener.onIncomingClusterState(incomingState);
lastSeenClusterState = incomingState;
}
} finally {
@ -411,56 +400,22 @@ public class PublishClusterStateAction extends AbstractComponent {
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}
// package private for testing
/**
* does simple sanity check of the incoming cluster state. Throws an exception on rejections.
*/
void validateIncomingState(ClusterState incomingState, ClusterState lastSeenClusterState) {
final ClusterName incomingClusterName = incomingState.getClusterName();
if (!incomingClusterName.equals(this.clusterName)) {
logger.warn("received cluster state from [{}] which is also master but with a different cluster name [{}]",
incomingState.nodes().getMasterNode(), incomingClusterName);
throw new IllegalStateException("received state from a node that is not part of the cluster");
}
final ClusterState clusterState = clusterStateSupplier.get();
if (clusterState.nodes().getLocalNode().equals(incomingState.nodes().getLocalNode()) == false) {
logger.warn("received a cluster state from [{}] and not part of the cluster, should not happen",
incomingState.nodes().getMasterNode());
throw new IllegalStateException("received state with a local node that does not match the current local node");
}
if (ZenDiscovery.shouldIgnoreOrRejectNewClusterState(logger, clusterState, incomingState)) {
String message = String.format(
Locale.ROOT,
"rejecting cluster state version [%d] uuid [%s] received from [%s]",
incomingState.version(),
incomingState.stateUUID(),
incomingState.nodes().getMasterNodeId()
);
logger.warn(message);
throw new IllegalStateException(message);
}
}
protected void handleCommitRequest(CommitClusterStateRequest request, final TransportChannel channel) {
final ClusterState state = pendingStatesQueue.markAsCommitted(request.stateUUID,
new PendingClusterStatesQueue.StateProcessedListener() {
incomingClusterStateListener.onClusterStateCommitted(request.stateUUID, new ActionListener<Void>() {
@Override
public void onNewClusterStateProcessed() {
public void onResponse(Void ignore) {
try {
// send a response to the master to indicate that this cluster state has been processed post committing it.
channel.sendResponse(TransportResponse.Empty.INSTANCE);
} catch (Exception e) {
logger.debug("failed to send response on cluster state processed", e);
onNewClusterStateFailed(e);
onFailure(e);
}
}
@Override
public void onNewClusterStateFailed(Exception e) {
public void onFailure(Exception e) {
try {
channel.sendResponse(e);
} catch (Exception inner) {
@ -469,10 +424,6 @@ public class PublishClusterStateAction extends AbstractComponent {
}
}
});
if (state != null) {
newPendingClusterStatelistener.onNewClusterState("master " + state.nodes().getMasterNode() +
" committed version [" + state.version() + "]");
}
}
private class SendClusterStateRequestHandler implements TransportRequestHandler<BytesTransportRequest> {

91
core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
View File

@ -25,6 +25,7 @@ import org.apache.logging.log4j.util.Supplier;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
@ -56,6 +57,7 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.discovery.Discovery;
import org.elasticsearch.discovery.DiscoverySettings;
import org.elasticsearch.discovery.DiscoveryStats;
import org.elasticsearch.discovery.zen.PublishClusterStateAction.IncomingClusterStateListener;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.EmptyTransportResponseHandler;
import org.elasticsearch.transport.TransportChannel;
@ -82,7 +84,7 @@ import java.util.stream.Collectors;
import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
public class ZenDiscovery extends AbstractLifecycleComponent implements Discovery, PingContextProvider {
public class ZenDiscovery extends AbstractLifecycleComponent implements Discovery, PingContextProvider, IncomingClusterStateListener {
public static final Setting<TimeValue> PING_TIMEOUT_SETTING =
Setting.positiveTimeSetting("discovery.zen.ping_timeout", timeValueSeconds(3), Property.NodeScope);
@ -104,6 +106,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
Property.NodeScope);
public static final Setting<Boolean> MASTER_ELECTION_IGNORE_NON_MASTER_PINGS_SETTING =
Setting.boolSetting("discovery.zen.master_election.ignore_non_master_pings", false, Property.NodeScope);
public static final Setting<Integer> MAX_PENDING_CLUSTER_STATES_SETTING =
Setting.intSetting("discovery.zen.publish.max_pending_cluster_states", 25, 1, Property.NodeScope);
public static final String DISCOVERY_REJOIN_ACTION_NAME = "internal:discovery/zen/rejoin";
@ -139,6 +143,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
private final JoinThreadControl joinThreadControl;
private final PendingClusterStatesQueue pendingStatesQueue;
private final NodeJoinController nodeJoinController;
private final NodeRemovalClusterStateTaskExecutor nodeRemovalExecutor;
@ -197,16 +203,15 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
this.masterFD.addListener(new MasterNodeFailureListener());
this.nodesFD = new NodesFaultDetection(settings, threadPool, transportService, clusterName);
this.nodesFD.addListener(new NodeFaultDetectionListener());
this.pendingStatesQueue = new PendingClusterStatesQueue(logger, MAX_PENDING_CLUSTER_STATES_SETTING.get(settings));
this.publishClusterState =
new PublishClusterStateAction(
settings,
transportService,
namedWriteableRegistry,
this::clusterState,
new NewPendingClusterStateListener(),
discoverySettings,
clusterName);
this,
discoverySettings);
this.membership = new MembershipAction(settings, transportService, new MembershipListener());
this.joinThreadControl = new JoinThreadControl();
@ -311,7 +316,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
throw new FailedToCommitClusterStateException("state was mutated while calculating new CS update");
}
publishClusterState.pendingStatesQueue().addPending(newState);
pendingStatesQueue.addPending(newState);
try {
publishClusterState.publish(clusterChangedEvent, electMaster.minimumMasterNodes(), ackListener);
@ -321,7 +326,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
newState.version(), electMaster.minimumMasterNodes());
synchronized (stateMutex) {
publishClusterState.pendingStatesQueue().failAllStatesAndClear(
pendingStatesQueue.failAllStatesAndClear(
new ElasticsearchException("failed to publish cluster state"));
rejoin("zen-disco-failed-to-publish");
@ -332,7 +337,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
final DiscoveryNode localNode = newState.getNodes().getLocalNode();
final CountDownLatch latch = new CountDownLatch(1);
final AtomicBoolean processedOrFailed = new AtomicBoolean();
publishClusterState.pendingStatesQueue().markAsCommitted(newState.stateUUID(),
pendingStatesQueue.markAsCommitted(newState.stateUUID(),
new PendingClusterStatesQueue.StateProcessedListener() {
@Override
public void onNewClusterStateProcessed() {
@ -391,7 +396,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
@Override
public DiscoveryStats stats() {
PendingClusterStateStats queueStats = publishClusterState.pendingStatesQueue().stats();
PendingClusterStateStats queueStats = pendingStatesQueue.stats();
return new DiscoveryStats(queueStats);
}
@ -409,11 +414,11 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
// used for testing
public ClusterState[] pendingClusterStates() {
return publishClusterState.pendingStatesQueue().pendingClusterStates();
return pendingStatesQueue.pendingClusterStates();
}
PendingClusterStatesQueue pendingClusterStatesQueue() {
return publishClusterState.pendingStatesQueue();
return pendingStatesQueue;
}
/**
@ -703,7 +708,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
synchronized (stateMutex) {
if (localNodeMaster() == false && masterNode.equals(committedState.get().nodes().getMasterNode())) {
// flush any pending cluster states from old master, so it will not be set as master again
publishClusterState.pendingStatesQueue().failAllStatesAndClear(new ElasticsearchException("master left [{}]", reason));
pendingStatesQueue.failAllStatesAndClear(new ElasticsearchException("master left [{}]", reason));
rejoin("master left (reason = " + reason + ")");
}
}
@ -713,7 +718,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
boolean processNextCommittedClusterState(String reason) {
assert Thread.holdsLock(stateMutex);
final ClusterState newClusterState = publishClusterState.pendingStatesQueue().getNextClusterStateToProcess();
final ClusterState newClusterState = pendingStatesQueue.getNextClusterStateToProcess();
final ClusterState currentState = committedState.get();
final ClusterState adaptedNewClusterState;
// all pending states have been processed
@ -742,7 +747,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
}
} catch (Exception e) {
try {
publishClusterState.pendingStatesQueue().markAsFailed(newClusterState, e);
pendingStatesQueue.markAsFailed(newClusterState, e);
} catch (Exception inner) {
inner.addSuppressed(e);
logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected exception while failing [{}]", reason), inner);
@ -811,7 +816,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
@Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
try {
publishClusterState.pendingStatesQueue().markAsProcessed(newClusterState);
pendingStatesQueue.markAsProcessed(newClusterState);
} catch (Exception e) {
onFailure(source, e);
}
@ -823,7 +828,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
try {
// TODO: use cluster state uuid instead of full cluster state so that we don't keep reference to CS around
// for too long.
publishClusterState.pendingStatesQueue().markAsFailed(newClusterState, e);
pendingStatesQueue.markAsFailed(newClusterState, e);
} catch (Exception inner) {
inner.addSuppressed(e);
logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected exception while failing [{}]", reason), inner);
@ -1066,16 +1071,64 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
}
}
private class NewPendingClusterStateListener implements PublishClusterStateAction.NewPendingClusterStateListener {
@Override
public void onIncomingClusterState(ClusterState incomingState) {
validateIncomingState(logger, incomingState, committedState.get());
pendingStatesQueue.addPending(incomingState);
}
@Override
public void onNewClusterState(String reason) {
public void onClusterStateCommitted(String stateUUID, ActionListener<Void> processedListener) {
final ClusterState state = pendingStatesQueue.markAsCommitted(stateUUID,
new PendingClusterStatesQueue.StateProcessedListener() {
@Override
public void onNewClusterStateProcessed() {
processedListener.onResponse(null);
}
@Override
public void onNewClusterStateFailed(Exception e) {
processedListener.onFailure(e);
}
});
if (state != null) {
synchronized (stateMutex) {
processNextCommittedClusterState(reason);
processNextCommittedClusterState("master " + state.nodes().getMasterNode() +
" committed version [" + state.version() + "]");
}
}
}
/**
* does simple sanity check of the incoming cluster state. Throws an exception on rejections.
*/
static void validateIncomingState(Logger logger, ClusterState incomingState, ClusterState lastState) {
final ClusterName incomingClusterName = incomingState.getClusterName();
if (!incomingClusterName.equals(lastState.getClusterName())) {
logger.warn("received cluster state from [{}] which is also master but with a different cluster name [{}]",
incomingState.nodes().getMasterNode(), incomingClusterName);
throw new IllegalStateException("received state from a node that is not part of the cluster");
}
if (lastState.nodes().getLocalNode().equals(incomingState.nodes().getLocalNode()) == false) {
logger.warn("received a cluster state from [{}] and not part of the cluster, should not happen",
incomingState.nodes().getMasterNode());
throw new IllegalStateException("received state with a local node that does not match the current local node");
}
if (shouldIgnoreOrRejectNewClusterState(logger, lastState, incomingState)) {
String message = String.format(
Locale.ROOT,
"rejecting cluster state version [%d] uuid [%s] received from [%s]",
incomingState.version(),
incomingState.stateUUID(),
incomingState.nodes().getMasterNodeId()
);
logger.warn(message);
throw new IllegalStateException(message);
}
}
private class MembershipListener implements MembershipAction.MembershipListener {
@Override
public void onJoin(DiscoveryNode node, MembershipAction.JoinCallback callback) {

32
core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
View File

@ -33,6 +33,7 @@ import org.elasticsearch.cluster.action.shard.ShardStateAction;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.routing.AllocationId;
import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
import org.elasticsearch.cluster.routing.RecoverySource.Type;
import org.elasticsearch.cluster.routing.RoutingNode;
@ -551,17 +552,15 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
try {
shard.updateRoutingEntry(shardRouting);
if (shardRouting.primary()) {
IndexShardRoutingTable indexShardRoutingTable = routingTable.shardRoutingTable(shardRouting.shardId());
Set<String> activeIds = indexShardRoutingTable.activeShards().stream()
// filter to shards that track seq# and should be taken into consideration for checkpoint tracking
// shards on old nodes will go through a file based recovery which will also transfer seq# information.
.filter(sr -> nodes.get(sr.currentNodeId()).getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED))
.map(r -> r.allocationId().getId())
.collect(Collectors.toSet());
Set<String> initializingIds = indexShardRoutingTable.getAllInitializingShards().stream()
.filter(sr -> nodes.get(sr.currentNodeId()).getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED))
.map(r -> r.allocationId().getId())
.collect(Collectors.toSet());
final IndexShardRoutingTable indexShardRoutingTable = routingTable.shardRoutingTable(shardRouting.shardId());
/*
* Filter to shards that track sequence numbers and should be taken into consideration for checkpoint tracking. Shards on
* old nodes will go through a file-based recovery which will also transfer sequence number information.
*/
final Set<String> activeIds =
allocationIdsForShardsOnNodesThatUnderstandSeqNos(indexShardRoutingTable.activeShards(), nodes);
final Set<String> initializingIds =
allocationIdsForShardsOnNodesThatUnderstandSeqNos(indexShardRoutingTable.getAllInitializingShards(), nodes);
shard.updateAllocationIdsFromMaster(activeIds, initializingIds);
}
} catch (Exception e) {
@ -586,6 +585,17 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
}
}
private Set<String> allocationIdsForShardsOnNodesThatUnderstandSeqNos(
final List<ShardRouting> shardRoutings,
final DiscoveryNodes nodes) {
return shardRoutings
.stream()
.filter(sr -> nodes.get(sr.currentNodeId()).getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED))
.map(ShardRouting::allocationId)
.map(AllocationId::getId)
.collect(Collectors.toSet());
}
/**
* Finds the routing source node for peer recovery, return null if its not found. Note, this method expects the shard
* routing to *require* peer recovery, use {@link ShardRouting#recoverySource()} to check if its needed or not.

12
core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
View File

@ -299,19 +299,19 @@ public class JvmInfo implements Writeable, ToXContent {
public int versionAsInteger() {
try {
int i = 0;
String sVersion = "";
StringBuilder sVersion = new StringBuilder();
for (; i < version.length(); i++) {
if (!Character.isDigit(version.charAt(i)) && version.charAt(i) != '.') {
break;
}
if (version.charAt(i) != '.') {
sVersion += version.charAt(i);
sVersion.append(version.charAt(i));
}
}
if (i == 0) {
return -1;
}
return Integer.parseInt(sVersion);
return Integer.parseInt(sVersion.toString());
} catch (Exception e) {
return -1;
}
@ -320,19 +320,19 @@ public class JvmInfo implements Writeable, ToXContent {
public int versionUpdatePack() {
try {
int i = 0;
String sVersion = "";
StringBuilder sVersion = new StringBuilder();
for (; i < version.length(); i++) {
if (!Character.isDigit(version.charAt(i)) && version.charAt(i) != '.') {
break;
}
if (version.charAt(i) != '.') {
sVersion += version.charAt(i);
sVersion.append(version.charAt(i));
}
}
if (i == 0) {
return -1;
}
Integer.parseInt(sVersion);
Integer.parseInt(sVersion.toString());
int from;
if (version.charAt(i) == '_') {
// 1.7.0_4

1
core/src/main/java/org/elasticsearch/node/Node.java
View File

@ -327,7 +327,6 @@ public class Node implements Closeable {
final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool);
final ScriptModule scriptModule = ScriptModule.create(settings, pluginsService.filterPlugins(ScriptPlugin.class));
AnalysisModule analysisModule = new AnalysisModule(this.environment, pluginsService.filterPlugins(AnalysisPlugin.class));
additionalSettings.addAll(scriptModule.getSettings());
// this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool
// so we might be late here already
final SettingsModule settingsModule = new SettingsModule(this.settings, additionalSettings, additionalSettingsFilter);

18
core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
View File

@ -85,12 +85,12 @@ public abstract class BaseRestHandler extends AbstractComponent implements RestH
final Set<String> invalids,
final Set<String> candidates,
final String detail) {
String message = String.format(
StringBuilder message = new StringBuilder(String.format(
Locale.ROOT,
"request [%s] contains unrecognized %s%s: ",
request.path(),
detail,
invalids.size() > 1 ? "s" : "");
invalids.size() > 1 ? "s" : ""));
boolean first = true;
for (final String invalid : invalids) {
final LevensteinDistance ld = new LevensteinDistance();
@ -108,17 +108,23 @@ public abstract class BaseRestHandler extends AbstractComponent implements RestH
else return a.v2().compareTo(b.v2());
});
if (first == false) {
message += ", ";
message.append(", ");
}
message += "[" + invalid + "]";
message.append("[").append(invalid).append("]");
final List<String> keys = scoredParams.stream().map(Tuple::v2).collect(Collectors.toList());
if (keys.isEmpty() == false) {
message += " -> did you mean " + (keys.size() == 1 ? "[" + keys.get(0) + "]" : "any of " + keys.toString()) + "?";
message.append(" -> did you mean ");
if (keys.size() == 1) {
message.append("[").append(keys.get(0)).append("]");
} else {
message.append("any of ").append(keys.toString());
}
message.append("?");
}
first = false;
}
return message;
return message.toString();
}
/**

150
core/src/main/java/org/elasticsearch/script/ScriptModes.java
View File

@ -1,150 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.function.Function;
/**
* Holds the boolean indicating the enabled mode for each of the different scripting languages available, each script source and each
* scripted operation.
*/
public class ScriptModes {
private static final String SCRIPT_SETTINGS_PREFIX = "script";
private static final String ENGINE_SETTINGS_PREFIX = "script.engine";
final Map<String, Boolean> scriptEnabled;
private static final Setting<List<String>> TYPES_ALLOWED_SETTING =
Setting.listSetting("script.types_allowed", Collections.emptyList(), Function.identity(), Setting.Property.NodeScope);
private static final Setting<List<String>> CONTEXTS_ALLOWED_SETTING =
Setting.listSetting("script.contexts_allowed", Collections.emptyList(), Function.identity(), Setting.Property.NodeScope);
private final Set<String> typesAllowed;
private final Set<String> contextsAllowed;
ScriptModes(ScriptContextRegistry scriptContextRegistry, ScriptSettings scriptSettings, Settings settings) {
HashMap<String, Boolean> scriptModes = new HashMap<>();
for (Setting<Boolean> scriptModeSetting : scriptSettings.getScriptLanguageSettings()) {
scriptModes.put(scriptModeSetting.getKey(), scriptModeSetting.get(settings));
}
this.scriptEnabled = Collections.unmodifiableMap(scriptModes);
typesAllowed = TYPES_ALLOWED_SETTING.exists(settings) ? new HashSet<>() : null;
if (typesAllowed != null) {
for (String settingType : TYPES_ALLOWED_SETTING.get(settings)) {
boolean found = false;
for (ScriptType scriptType : ScriptType.values()) {
if (scriptType.getName().equals(settingType)) {
found = true;
typesAllowed.add(settingType);
break;
}
}
if (!found) {
throw new IllegalArgumentException(
"unknown script type [" + settingType + "] found in setting [" + TYPES_ALLOWED_SETTING.getKey() + "].");
}
}
}
contextsAllowed = CONTEXTS_ALLOWED_SETTING.exists(settings) ? new HashSet<>() : null;
if (contextsAllowed != null) {
for (String settingContext : CONTEXTS_ALLOWED_SETTING.get(settings)) {
if (scriptContextRegistry.isSupportedContext(settingContext)) {
contextsAllowed.add(settingContext);
} else {
throw new IllegalArgumentException(
"unknown script context [" + settingContext + "] found in setting [" + CONTEXTS_ALLOWED_SETTING.getKey() + "].");
}
}
}
}
/**
* Returns the script mode for a script of a certain written in a certain language,
* of a certain type and executing as part of a specific operation/api.
*
* @param lang the language that the script is written in
* @param scriptType the type of the script
* @param scriptContext the operation that requires the execution of the script
* @return whether scripts are enabled (true) or disabled (false)
*/
public boolean getScriptEnabled(String lang, ScriptType scriptType, ScriptContext scriptContext) {
if (typesAllowed != null && typesAllowed.contains(scriptType.getName()) == false) {
throw new IllegalArgumentException("[" + scriptType.getName() + "] scripts cannot be executed");
}
if (contextsAllowed != null && contextsAllowed.contains(scriptContext.getKey()) == false) {
throw new IllegalArgumentException("[" + scriptContext.getKey() + "] scripts cannot be executed");
}
Boolean scriptMode = scriptEnabled.get(getKey(lang, scriptType, scriptContext));
if (scriptMode == null) {
throw new IllegalArgumentException("script mode not found for lang [" + lang + "], script_type [" + scriptType + "], operation [" + scriptContext.getKey() + "]");
}
return scriptMode;
}
static String operationKey(ScriptContext scriptContext) {
return SCRIPT_SETTINGS_PREFIX + "." + scriptContext.getKey();
}
static String sourceKey(ScriptType scriptType) {
return SCRIPT_SETTINGS_PREFIX + "." + scriptType.getName();
}
static String getGlobalKey(String lang, ScriptType scriptType) {
return ENGINE_SETTINGS_PREFIX + "." + lang + "." + scriptType;
}
static String getKey(String lang, ScriptType scriptType, ScriptContext scriptContext) {
return ENGINE_SETTINGS_PREFIX + "." + lang + "." + scriptType + "." + scriptContext.getKey();
}
@Override
public String toString() {
//order settings by key before printing them out, for readability
TreeMap<String, Boolean> scriptModesTreeMap = new TreeMap<>();
scriptModesTreeMap.putAll(scriptEnabled);
StringBuilder stringBuilder = new StringBuilder();
for (Map.Entry<String, Boolean> stringScriptModeEntry : scriptModesTreeMap.entrySet()) {
stringBuilder.append(stringScriptModeEntry.getKey()).append(": ").append(stringScriptModeEntry.getValue()).append("\n");
}
return stringBuilder.toString();
}
}

18
core/src/main/java/org/elasticsearch/script/ScriptModule.java
View File

@ -20,24 +20,18 @@
package org.elasticsearch.script;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.plugins.ScriptPlugin;
import org.elasticsearch.watcher.ResourceWatcherService;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* Manages building {@link ScriptService} and {@link ScriptSettings} from a list of plugins.
* Manages building {@link ScriptService}.
*/
public class ScriptModule {
private final ScriptSettings scriptSettings;
private final ScriptService scriptService;
/**
@ -59,21 +53,13 @@ public class ScriptModule {
List<ScriptContext.Plugin> customScriptContexts) {
ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customScriptContexts);
ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(scriptEngines);
scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
try {
scriptService = new ScriptService(settings, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
scriptService = new ScriptService(settings, scriptEngineRegistry, scriptContextRegistry);
} catch (IOException e) {
throw new RuntimeException("Couldn't setup ScriptService", e);
}
}
/**
* Extra settings for scripts.
*/
public List<Setting<?>> getSettings() {
return scriptSettings.getSettings();
}
/**
* Service responsible for managing scripts.
*/

161
core/src/main/java/org/elasticsearch/script/ScriptService.java
View File

@ -19,12 +19,6 @@
package org.elasticsearch.script;
import java.io.Closeable;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import java.util.Objects;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ResourceNotFoundException;
import org.elasticsearch.action.ActionListener;
@ -41,7 +35,6 @@ import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.breaker.CircuitBreakingException;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.cache.Cache;
import org.elasticsearch.common.cache.CacheBuilder;
import org.elasticsearch.common.cache.RemovalListener;
@ -55,6 +48,16 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.search.lookup.SearchLookup;
import org.elasticsearch.template.CompiledTemplate;
import java.io.Closeable;
import java.io.IOException;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
public class ScriptService extends AbstractComponent implements Closeable, ClusterStateListener {
static final String DISABLE_DYNAMIC_SCRIPTING_SETTING = "script.disable_dynamic";
@ -68,11 +71,20 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
public static final Setting<Integer> SCRIPT_MAX_COMPILATIONS_PER_MINUTE =
Setting.intSetting("script.max_compilations_per_minute", 15, 0, Property.Dynamic, Property.NodeScope);
public static final String ALLOW_NONE = "none";
public static final Setting<List<String>> TYPES_ALLOWED_SETTING =
Setting.listSetting("script.types_allowed", Collections.emptyList(), Function.identity(), Setting.Property.NodeScope);
public static final Setting<List<String>> CONTEXTS_ALLOWED_SETTING =
Setting.listSetting("script.contexts_allowed", Collections.emptyList(), Function.identity(), Setting.Property.NodeScope);
private final Set<String> typesAllowed;
private final Set<String> contextsAllowed;
private final Map<String, ScriptEngine> engines;
private final Cache<CacheKey, CompiledScript> cache;
private final ScriptModes scriptModes;
private final ScriptContextRegistry scriptContextRegistry;
private final ScriptMetrics scriptMetrics = new ScriptMetrics();
@ -84,18 +96,87 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
private double scriptsPerMinCounter;
private double compilesAllowedPerNano;
public ScriptService(Settings settings, ScriptEngineRegistry scriptEngineRegistry,
ScriptContextRegistry scriptContextRegistry, ScriptSettings scriptSettings) throws IOException {
public ScriptService(Settings settings, ScriptEngineRegistry scriptEngineRegistry, ScriptContextRegistry scriptContextRegistry) throws IOException {
super(settings);
Objects.requireNonNull(settings);
Objects.requireNonNull(scriptEngineRegistry);
Objects.requireNonNull(scriptContextRegistry);
Objects.requireNonNull(scriptSettings);
if (Strings.hasLength(settings.get(DISABLE_DYNAMIC_SCRIPTING_SETTING))) {
throw new IllegalArgumentException(DISABLE_DYNAMIC_SCRIPTING_SETTING + " is not a supported setting, replace with fine-grained script settings. \n" +
"Dynamic scripts can be enabled for all languages and all operations by replacing `script.disable_dynamic: false` with `script.inline: true` and `script.stored: true` in elasticsearch.yml");
"Dynamic scripts can be enabled for all languages and all operations not using `script.disable_dynamic: false` in elasticsearch.yml");
}
this.typesAllowed = TYPES_ALLOWED_SETTING.exists(settings) ? new HashSet<>() : null;
if (this.typesAllowed != null) {
List<String> typesAllowedList = TYPES_ALLOWED_SETTING.get(settings);
if (typesAllowedList.isEmpty()) {
throw new IllegalArgumentException(
"must specify at least one script type or none for setting [" + TYPES_ALLOWED_SETTING.getKey() + "].");
}
for (String settingType : typesAllowedList) {
if (ALLOW_NONE.equals(settingType)) {
if (typesAllowedList.size() != 1) {
throw new IllegalArgumentException("cannot specify both [" + ALLOW_NONE + "]" +
" and other script types for setting [" + TYPES_ALLOWED_SETTING.getKey() + "].");
} else {
break;
}
}
boolean found = false;
for (ScriptType scriptType : ScriptType.values()) {
if (scriptType.getName().equals(settingType)) {
found = true;
this.typesAllowed.add(settingType);
break;
}
}
if (found == false) {
throw new IllegalArgumentException(
"unknown script type [" + settingType + "] found in setting [" + TYPES_ALLOWED_SETTING.getKey() + "].");
}
}
}
this.contextsAllowed = CONTEXTS_ALLOWED_SETTING.exists(settings) ? new HashSet<>() : null;
if (this.contextsAllowed != null) {
List<String> contextsAllowedList = CONTEXTS_ALLOWED_SETTING.get(settings);
if (contextsAllowedList.isEmpty()) {
throw new IllegalArgumentException(
"must specify at least one script context or none for setting [" + CONTEXTS_ALLOWED_SETTING.getKey() + "].");
}
for (String settingContext : contextsAllowedList) {
if (ALLOW_NONE.equals(settingContext)) {
if (contextsAllowedList.size() != 1) {
throw new IllegalArgumentException("cannot specify both [" + ALLOW_NONE + "]" +
" and other script contexts for setting [" + CONTEXTS_ALLOWED_SETTING.getKey() + "].");
} else {
break;
}
}
if (scriptContextRegistry.isSupportedContext(settingContext)) {
this.contextsAllowed.add(settingContext);
} else {
throw new IllegalArgumentException(
"unknown script context [" + settingContext + "] found in setting [" + CONTEXTS_ALLOWED_SETTING.getKey() + "].");
}
}
}
this.scriptContextRegistry = scriptContextRegistry;
int cacheMaxSize = SCRIPT_CACHE_SIZE_SETTING.get(settings);
CacheBuilder<CacheKey, CompiledScript> cacheBuilder = CacheBuilder.builder();
@ -110,8 +191,9 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
logger.debug("using script cache with max_size [{}], expire [{}]", cacheMaxSize, cacheExpire);
this.cache = cacheBuilder.removalListener(new ScriptCacheRemovalListener()).build();
this.engines = scriptEngineRegistry.getRegisteredLanguages();
this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, settings);
this.lastInlineCompileTime = System.nanoTime();
this.setMaxCompilationsPerMinute(SCRIPT_MAX_COMPILATIONS_PER_MINUTE.get(settings));
}
@ -194,9 +276,16 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
ScriptEngine scriptEngine = getEngine(lang);
if (canExecuteScript(lang, type, scriptContext) == false) {
throw new IllegalStateException("scripts of type [" + script.getType() + "]," +
" operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are disabled");
if (isTypeEnabled(type) == false) {
throw new IllegalArgumentException("cannot execute [" + type + "] scripts");
}
if (scriptContextRegistry.isSupportedContext(scriptContext.getKey()) == false) {
throw new IllegalArgumentException("script context [" + scriptContext.getKey() + "] not supported");
}
if (isContextEnabled(scriptContext) == false) {
throw new IllegalArgumentException("cannot execute scripts using [" + scriptContext.getKey() + "] context");
}
if (logger.isTraceEnabled()) {
@ -288,6 +377,18 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
return engines.containsKey(lang);
}
public boolean isTypeEnabled(ScriptType scriptType) {
return typesAllowed == null || typesAllowed.contains(scriptType.getName());
}
public boolean isContextEnabled(ScriptContext scriptContext) {
return contextsAllowed == null || contextsAllowed.contains(scriptContext.getKey());
}
public boolean isAnyContextEnabled() {
return contextsAllowed == null || contextsAllowed.isEmpty() == false;
}
StoredScriptSource getScriptFromClusterState(String id, String lang) {
if (lang != null && isLangSupported(lang) == false) {
throw new IllegalArgumentException("unable to get stored script with unsupported lang [" + lang + "]");
@ -328,16 +429,19 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
try {
ScriptEngine scriptEngine = getEngine(source.getLang());
if (isAnyScriptContextEnabled(source.getLang(), ScriptType.STORED)) {
if (isTypeEnabled(ScriptType.STORED) == false) {
throw new IllegalArgumentException(
"cannot put [" + ScriptType.STORED + "] script, [" + ScriptType.STORED + "] scripts are not enabled");
} else if (isAnyContextEnabled() == false) {
throw new IllegalArgumentException(
"cannot put [" + ScriptType.STORED + "] script, no script contexts are enabled");
} else {
Object compiled = scriptEngine.compile(request.id(), source.getCode(), Collections.emptyMap());
if (compiled == null) {
throw new IllegalArgumentException("failed to parse/compile stored script [" + request.id() + "]" +
(source.getCode() == null ? "" : " using code [" + source.getCode() + "]"));
}
} else {
throw new IllegalArgumentException(
"cannot put stored script [" + request.id() + "], stored scripts cannot be run under any context");
}
} catch (ScriptException good) {
throw good;
@ -422,23 +526,6 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
return getEngine(compiledScript.lang()).search(compiledScript, lookup, params);
}
private boolean isAnyScriptContextEnabled(String lang, ScriptType scriptType) {
for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) {
if (canExecuteScript(lang, scriptType, scriptContext)) {
return true;
}
}
return false;
}
private boolean canExecuteScript(String lang, ScriptType scriptType, ScriptContext scriptContext) {
assert lang != null;
if (scriptContextRegistry.isSupportedContext(scriptContext.getKey()) == false) {
throw new IllegalArgumentException("script context [" + scriptContext.getKey() + "] not supported");
}
return scriptModes.getScriptEnabled(lang, scriptType, scriptContext);
}
public ScriptStats stats() {
return scriptMetrics.stats();
}

155
core/src/main/java/org/elasticsearch/script/ScriptSettings.java
View File

@ -1,155 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
public class ScriptSettings {
private static final Map<ScriptType, Setting<Boolean>> SCRIPT_TYPE_SETTING_MAP;
static {
Map<ScriptType, Setting<Boolean>> scriptTypeSettingMap = new EnumMap<>(ScriptType.class);
for (ScriptType scriptType : ScriptType.values()) {
scriptTypeSettingMap.put(scriptType, Setting.boolSetting(
ScriptModes.sourceKey(scriptType),
scriptType.isDefaultEnabled(),
Property.NodeScope,
Property.Deprecated));
}
SCRIPT_TYPE_SETTING_MAP = Collections.unmodifiableMap(scriptTypeSettingMap);
}
private final Map<ScriptContext, Setting<Boolean>> scriptContextSettingMap;
private final List<Setting<Boolean>> scriptLanguageSettings;
public ScriptSettings(ScriptEngineRegistry scriptEngineRegistry, ScriptContextRegistry scriptContextRegistry) {
Map<ScriptContext, Setting<Boolean>> scriptContextSettingMap = contextSettings(scriptContextRegistry);
this.scriptContextSettingMap = Collections.unmodifiableMap(scriptContextSettingMap);
List<Setting<Boolean>> scriptLanguageSettings = languageSettings(SCRIPT_TYPE_SETTING_MAP, scriptContextSettingMap, scriptEngineRegistry, scriptContextRegistry);
this.scriptLanguageSettings = Collections.unmodifiableList(scriptLanguageSettings);
}
private static Map<ScriptContext, Setting<Boolean>> contextSettings(ScriptContextRegistry scriptContextRegistry) {
Map<ScriptContext, Setting<Boolean>> scriptContextSettingMap = new HashMap<>();
for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) {
scriptContextSettingMap.put(scriptContext,
Setting.boolSetting(ScriptModes.operationKey(scriptContext), false, Property.NodeScope, Property.Deprecated));
}
return scriptContextSettingMap;
}
private static List<Setting<Boolean>> languageSettings(Map<ScriptType, Setting<Boolean>> scriptTypeSettingMap,
Map<ScriptContext, Setting<Boolean>> scriptContextSettingMap,
ScriptEngineRegistry scriptEngineRegistry,
ScriptContextRegistry scriptContextRegistry) {
final List<Setting<Boolean>> scriptModeSettings = new ArrayList<>();
for (final Class<? extends ScriptEngine> scriptEngineService : scriptEngineRegistry.getRegisteredScriptEngineServices()) {
final String language = scriptEngineRegistry.getLanguage(scriptEngineService);
for (final ScriptType scriptType : ScriptType.values()) {
// Top level, like "script.engine.groovy.inline"
final boolean defaultNonFileScriptMode = scriptEngineRegistry.getDefaultInlineScriptEnableds().get(language);
boolean defaultLangAndType = defaultNonFileScriptMode;
// Files are treated differently because they are never default-deny
final boolean defaultIfNothingSet = defaultLangAndType;
Function<Settings, String> defaultLangAndTypeFn = settings -> {
final Setting<Boolean> globalTypeSetting = scriptTypeSettingMap.get(scriptType);
final Setting<Boolean> langAndTypeSetting = Setting.boolSetting(ScriptModes.getGlobalKey(language, scriptType),
defaultIfNothingSet, Property.NodeScope, Property.Deprecated);
if (langAndTypeSetting.exists(settings)) {
// fine-grained e.g. script.engine.groovy.inline
return langAndTypeSetting.get(settings).toString();
} else if (globalTypeSetting.exists(settings)) {
// global type - script.inline
return globalTypeSetting.get(settings).toString();
} else {
return Boolean.toString(defaultIfNothingSet);
}
};
// Setting for something like "script.engine.groovy.inline"
final Setting<Boolean> langAndTypeSetting = Setting.boolSetting(ScriptModes.getGlobalKey(language, scriptType),
defaultLangAndTypeFn, Property.NodeScope, Property.Deprecated);
scriptModeSettings.add(langAndTypeSetting);
for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) {
final String langAndTypeAndContextName = ScriptModes.getKey(language, scriptType, scriptContext);
// A function that, given a setting, will return what the default should be. Since the fine-grained script settings
// read from a bunch of different places this is implemented in this way.
Function<Settings, String> defaultSettingFn = settings -> {
final Setting<Boolean> globalOpSetting = scriptContextSettingMap.get(scriptContext);
final Setting<Boolean> globalTypeSetting = scriptTypeSettingMap.get(scriptType);
final Setting<Boolean> langAndTypeAndContextSetting = Setting.boolSetting(langAndTypeAndContextName,
defaultIfNothingSet, Property.NodeScope, Property.Deprecated);
// fallback logic for script mode settings
if (langAndTypeAndContextSetting.exists(settings)) {
// like: "script.engine.groovy.inline.aggs: true"
return langAndTypeAndContextSetting.get(settings).toString();
} else if (langAndTypeSetting.exists(settings)) {
// like: "script.engine.groovy.inline: true"
return langAndTypeSetting.get(settings).toString();
} else if (globalOpSetting.exists(settings)) {
// like: "script.aggs: true"
return globalOpSetting.get(settings).toString();
} else if (globalTypeSetting.exists(settings)) {
// like: "script.inline: true"
return globalTypeSetting.get(settings).toString();
} else {
// Nothing is set!
return Boolean.toString(defaultIfNothingSet);
}
};
// The actual setting for finest grained script settings
Setting<Boolean> setting =
Setting.boolSetting(langAndTypeAndContextName, defaultSettingFn, Property.NodeScope, Property.Deprecated);
scriptModeSettings.add(setting);
}
}
}
return scriptModeSettings;
}
public List<Setting<?>> getSettings() {
List<Setting<?>> settings = new ArrayList<>();
settings.addAll(SCRIPT_TYPE_SETTING_MAP.values());
settings.addAll(scriptContextSettingMap.values());
settings.addAll(scriptLanguageSettings);
return settings;
}
public Iterable<Setting<Boolean>> getScriptLanguageSettings() {
return scriptLanguageSettings;
}
}

4
core/src/main/java/org/elasticsearch/script/ScriptType.java
View File

@ -28,8 +28,8 @@ import java.io.IOException;
/**
* ScriptType represents the way a script is stored and retrieved from the {@link ScriptService}.
* It's also used to by {@link ScriptSettings} and {@link ScriptModes} to determine whether or not
* a {@link Script} is allowed to be executed based on both default and user-defined settings.
* It's also used to by {@link ScriptService} to determine whether or not a {@link Script} is
* allowed to be executed based on both default and user-defined settings.
*/
public enum ScriptType implements Writeable {

14
core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
View File

@ -369,8 +369,8 @@ public final class InternalDateHistogram extends InternalMultiBucketAggregation<
Bucket firstBucket = iter.hasNext() ? list.get(iter.nextIndex()) : null;
if (firstBucket == null) {
if (bounds.getMin() != null && bounds.getMax() != null) {
long key = bounds.getMin();
long max = bounds.getMax();
long key = bounds.getMin() + offset;
long max = bounds.getMax() + offset;
while (key <= max) {
iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs));
key = nextKey(key).longValue();
@ -378,7 +378,7 @@ public final class InternalDateHistogram extends InternalMultiBucketAggregation<
}
} else {
if (bounds.getMin() != null) {
long key = bounds.getMin();
long key = bounds.getMin() + offset;
if (key < firstBucket.key) {
while (key < firstBucket.key) {
iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs));
@ -405,12 +405,12 @@ public final class InternalDateHistogram extends InternalMultiBucketAggregation<
}
// finally, adding the empty buckets *after* the actual data (based on the extended_bounds.max requested by the user)
if (bounds != null && lastBucket != null && bounds.getMax() != null && bounds.getMax() > lastBucket.key) {
long key = emptyBucketInfo.rounding.nextRoundingValue(lastBucket.key);
long max = bounds.getMax();
if (bounds != null && lastBucket != null && bounds.getMax() != null && bounds.getMax() + offset > lastBucket.key) {
long key = nextKey(lastBucket.key).longValue();
long max = bounds.getMax() + offset;
while (key <= max) {
iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs));
key = emptyBucketInfo.rounding.nextRoundingValue(key);
key = nextKey(key).longValue();
}
}
}

5
core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java
View File

@ -76,11 +76,6 @@ public class SignificantLongTerms extends InternalMappedSignificantTerms<Signifi
return term;
}
@Override
public int compareTerm(SignificantTerms.Bucket other) {
return Long.compare(term, ((Number) other.getKey()).longValue());
}
@Override
public String getKeyAsString() {
return format.format(term);

6
core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java
View File

@ -28,7 +28,6 @@ import org.elasticsearch.search.aggregations.bucket.significant.heuristics.Signi
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -82,11 +81,6 @@ public class SignificantStringTerms extends InternalMappedSignificantTerms<Signi
return Double.parseDouble(termBytes.utf8ToString());
}
@Override
public int compareTerm(SignificantTerms.Bucket other) {
return termBytes.compareTo(((Bucket) other).termBytes);
}
@Override
public String getKeyAsString() {
return format.format(termBytes);

2
core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTerms.java
View File

@ -40,8 +40,6 @@ public interface SignificantTerms extends MultiBucketsAggregation, Iterable<Sign
long getSupersetSize();
long getSubsetSize();
int compareTerm(SignificantTerms.Bucket other);
}
@Override

11
core/src/test/java/org/elasticsearch/VersionTests.java
View File

@ -293,7 +293,6 @@ public class VersionTests extends ESTestCase {
if (maxBranchVersion == null) {
maxBranchVersions.put(branchName, v);
} else if (v.after(maxBranchVersion)) {
assertFalse("Version " + maxBranchVersion + " cannot be a snapshot because version " + v + " exists", VersionUtils.isSnapshot(maxBranchVersion));
maxBranchVersions.put(branchName, v);
}
@ -329,6 +328,16 @@ public class VersionTests extends ESTestCase {
assertTrue(isCompatible(Version.V_5_5_0_UNRELEASED, Version.V_6_0_0_alpha2_UNRELEASED));
assertFalse(isCompatible(Version.fromId(2000099), Version.V_6_0_0_alpha2_UNRELEASED));
assertFalse(isCompatible(Version.fromId(2000099), Version.V_5_0_0));
assertTrue(isCompatible(Version.fromString("6.0.0"), Version.fromString("7.0.0")));
if (Version.CURRENT.isRelease()) {
assertTrue(isCompatible(Version.CURRENT, Version.fromString("7.0.0")));
} else {
assertFalse(isCompatible(Version.CURRENT, Version.fromString("7.0.0")));
}
assertFalse("only compatible with the latest minor",
isCompatible(VersionUtils.getPreviousMinorVersion(), Version.fromString("7.0.0")));
assertFalse(isCompatible(Version.V_5_0_0, Version.fromString("6.0.0")));
assertFalse(isCompatible(Version.V_5_0_0, Version.fromString("7.0.0")));
}

7
core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java
View File

@ -45,7 +45,6 @@ import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptContextRegistry;
import org.elasticsearch.script.ScriptEngineRegistry;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptSettings;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.RandomObjects;
@ -64,7 +63,6 @@ import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonList;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
import static org.elasticsearch.common.xcontent.XContentHelper.update;
import static org.elasticsearch.script.MockScriptEngine.mockInlineScript;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
import static org.hamcrest.Matchers.arrayContaining;
@ -145,15 +143,12 @@ public class UpdateRequestTests extends ESTestCase {
final ScriptEngineRegistry scriptEngineRegistry =
new ScriptEngineRegistry(singletonList(engine));
final ScriptSettings scriptSettings =
new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
final ResourceWatcherService watcherService =
new ResourceWatcherService(baseSettings, null);
ScriptService scriptService = new ScriptService(
baseSettings,
scriptEngineRegistry,
scriptContextRegistry,
scriptSettings);
scriptContextRegistry);
final Settings settings = settings(Version.CURRENT).build();
updateHelper = new UpdateHelper(settings, scriptService);

127
core/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.discovery.zen;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterModule;
import org.elasticsearch.cluster.ClusterName;
@ -64,20 +65,17 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.emptyIterable;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasToString;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
@ -90,11 +88,12 @@ public class PublishClusterStateActionTests extends ESTestCase {
protected ThreadPool threadPool;
protected Map<String, MockNode> nodes = new HashMap<>();
public static class MockNode implements PublishClusterStateAction.NewPendingClusterStateListener {
public static class MockNode implements PublishClusterStateAction.IncomingClusterStateListener {
public final DiscoveryNode discoveryNode;
public final MockTransportService service;
public MockPublishAction action;
public final ClusterStateListener listener;
private final PendingClusterStatesQueue pendingStatesQueue;
public volatile ClusterState clusterState;
@ -108,6 +107,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
this.logger = logger;
this.clusterState = ClusterState.builder(CLUSTER_NAME).nodes(DiscoveryNodes.builder()
.add(discoveryNode).localNodeId(discoveryNode.getId()).build()).build();
this.pendingStatesQueue = new PendingClusterStatesQueue(logger, 25);
}
public MockNode setAsMaster() {
@ -128,8 +128,26 @@ public class PublishClusterStateActionTests extends ESTestCase {
}
@Override
public void onNewClusterState(String reason) {
ClusterState newClusterState = action.pendingStatesQueue().getNextClusterStateToProcess();
public void onIncomingClusterState(ClusterState incomingState) {
ZenDiscovery.validateIncomingState(logger, incomingState, clusterState);
pendingStatesQueue.addPending(incomingState);
}
public void onClusterStateCommitted(String stateUUID, ActionListener<Void> processedListener) {
final ClusterState state = pendingStatesQueue.markAsCommitted(stateUUID,
new PendingClusterStatesQueue.StateProcessedListener() {
@Override
public void onNewClusterStateProcessed() {
processedListener.onResponse(null);
}
@Override
public void onNewClusterStateFailed(Exception e) {
processedListener.onFailure(e);
}
});
if (state != null) {
ClusterState newClusterState = pendingStatesQueue.getNextClusterStateToProcess();
logger.debug("[{}] received version [{}], uuid [{}]",
discoveryNode.getName(), newClusterState.version(), newClusterState.stateUUID());
if (listener != null) {
@ -139,7 +157,8 @@ public class PublishClusterStateActionTests extends ESTestCase {
if (clusterState.nodes().getMasterNode() == null || newClusterState.supersedes(clusterState)) {
clusterState = newClusterState;
}
action.pendingStatesQueue().markAsProcessed(newClusterState);
pendingStatesQueue.markAsProcessed(newClusterState);
}
}
public DiscoveryNodes nodes() {
@ -168,7 +187,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
MockTransportService service = buildTransportService(settings, threadPool);
DiscoveryNode discoveryNode = service.getLocalDiscoNode();
MockNode node = new MockNode(discoveryNode, service, listener, logger);
node.action = buildPublishClusterStateAction(settings, service, () -> node.clusterState, node);
node.action = buildPublishClusterStateAction(settings, service, node);
final CountDownLatch latch = new CountDownLatch(nodes.size() * 2);
TransportConnectionListener waitForConnection = new TransportConnectionListener() {
@Override
@ -241,8 +260,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
private static MockPublishAction buildPublishClusterStateAction(
Settings settings,
MockTransportService transportService,
Supplier<ClusterState> clusterStateSupplier,
PublishClusterStateAction.NewPendingClusterStateListener listener
PublishClusterStateAction.IncomingClusterStateListener listener
) {
DiscoverySettings discoverySettings =
new DiscoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
@ -251,10 +269,8 @@ public class PublishClusterStateActionTests extends ESTestCase {
settings,
transportService,
namedWriteableRegistry,
clusterStateSupplier,
listener,
discoverySettings,
CLUSTER_NAME);
discoverySettings);
}
public void testSimpleClusterStatePublishing() throws Exception {
@ -607,86 +623,6 @@ public class PublishClusterStateActionTests extends ESTestCase {
}
}
public void testIncomingClusterStateValidation() throws Exception {
MockNode node = createMockNode("node");
logger.info("--> testing acceptances of any master when having no master");
ClusterState state = ClusterState.builder(node.clusterState)
.nodes(DiscoveryNodes.builder(node.nodes()).masterNodeId(randomAlphaOfLength(10))).incrementVersion().build();
node.action.validateIncomingState(state, null);
// now set a master node
node.clusterState = ClusterState.builder(node.clusterState)
.nodes(DiscoveryNodes.builder(node.nodes()).masterNodeId("master")).build();
logger.info("--> testing rejection of another master");
try {
node.action.validateIncomingState(state, node.clusterState);
fail("node accepted state from another master");
} catch (IllegalStateException OK) {
assertThat(OK.toString(), containsString("cluster state from a different master than the current one, rejecting"));
}
logger.info("--> test state from the current master is accepted");
node.action.validateIncomingState(ClusterState.builder(node.clusterState)
.nodes(DiscoveryNodes.builder(node.nodes()).masterNodeId("master")).incrementVersion().build(), node.clusterState);
logger.info("--> testing rejection of another cluster name");
try {
node.action.validateIncomingState(ClusterState.builder(new ClusterName(randomAlphaOfLength(10)))
.nodes(node.nodes()).build(), node.clusterState);
fail("node accepted state with another cluster name");
} catch (IllegalStateException OK) {
assertThat(OK.toString(), containsString("received state from a node that is not part of the cluster"));
}
logger.info("--> testing rejection of a cluster state with wrong local node");
try {
state = ClusterState.builder(node.clusterState)
.nodes(DiscoveryNodes.builder(node.nodes()).localNodeId("_non_existing_").build())
.incrementVersion().build();
node.action.validateIncomingState(state, node.clusterState);
fail("node accepted state with non-existence local node");
} catch (IllegalStateException OK) {
assertThat(OK.toString(), containsString("received state with a local node that does not match the current local node"));
}
try {
MockNode otherNode = createMockNode("otherNode");
state = ClusterState.builder(node.clusterState).nodes(
DiscoveryNodes.builder(node.nodes()).add(otherNode.discoveryNode).localNodeId(otherNode.discoveryNode.getId()).build()
).incrementVersion().build();
node.action.validateIncomingState(state, node.clusterState);
fail("node accepted state with existent but wrong local node");
} catch (IllegalStateException OK) {
assertThat(OK.toString(), containsString("received state with a local node that does not match the current local node"));
}
logger.info("--> testing acceptance of an old cluster state");
final ClusterState incomingState = node.clusterState;
node.clusterState = ClusterState.builder(node.clusterState).incrementVersion().build();
final IllegalStateException e =
expectThrows(IllegalStateException.class, () -> node.action.validateIncomingState(incomingState, node.clusterState));
final String message = String.format(
Locale.ROOT,
"rejecting cluster state version [%d] uuid [%s] received from [%s]",
incomingState.version(),
incomingState.stateUUID(),
incomingState.nodes().getMasterNodeId()
);
assertThat(e, hasToString("java.lang.IllegalStateException: " + message));
// an older version from a *new* master is also OK!
ClusterState previousState = ClusterState.builder(node.clusterState).incrementVersion().build();
state = ClusterState.builder(node.clusterState)
.nodes(DiscoveryNodes.builder(node.clusterState.nodes()).masterNodeId("_new_master_").build())
.build();
// remove the master of the node (but still have a previous cluster state with it)!
node.resetMasterId();
node.action.validateIncomingState(state, previousState);
}
public void testOutOfOrderCommitMessages() throws Throwable {
MockNode node = createMockNode("node").setAsMaster();
final CapturingTransportChannel channel = new CapturingTransportChannel();
@ -874,9 +810,8 @@ public class PublishClusterStateActionTests extends ESTestCase {
AtomicBoolean errorOnCommit = new AtomicBoolean();
public MockPublishAction(Settings settings, TransportService transportService, NamedWriteableRegistry namedWriteableRegistry,
Supplier<ClusterState> clusterStateSupplier, NewPendingClusterStateListener listener,
DiscoverySettings discoverySettings, ClusterName clusterName) {
super(settings, transportService, namedWriteableRegistry, clusterStateSupplier, listener, discoverySettings, clusterName);
IncomingClusterStateListener listener, DiscoverySettings discoverySettings) {
super(settings, transportService, namedWriteableRegistry, listener, discoverySettings);
}
@Override

92
core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java
View File

@ -67,6 +67,7 @@ import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@ -88,6 +89,7 @@ import static org.hamcrest.Matchers.arrayWithSize;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.emptyArray;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasToString;
public class ZenDiscoveryUnitTests extends ESTestCase {
@ -405,4 +407,94 @@ public class ZenDiscoveryUnitTests extends ESTestCase {
}
}
}
public void testIncomingClusterStateValidation() throws Exception {
ClusterName clusterName = new ClusterName("abc");
DiscoveryNodes.Builder currentNodes = DiscoveryNodes.builder().add(
new DiscoveryNode("a", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT)).localNodeId("a");
ClusterState previousState = ClusterState.builder(clusterName).nodes(currentNodes).build();
logger.info("--> testing acceptances of any master when having no master");
ClusterState state = ClusterState.builder(previousState)
.nodes(DiscoveryNodes.builder(previousState.nodes()).masterNodeId(randomAlphaOfLength(10))).incrementVersion().build();
ZenDiscovery.validateIncomingState(logger, state, previousState);
// now set a master node
previousState = state;
state = ClusterState.builder(previousState)
.nodes(DiscoveryNodes.builder(previousState.nodes()).masterNodeId("master")).build();
logger.info("--> testing rejection of another master");
try {
ZenDiscovery.validateIncomingState(logger, state, previousState);
fail("node accepted state from another master");
} catch (IllegalStateException OK) {
assertThat(OK.toString(), containsString("cluster state from a different master than the current one, rejecting"));
}
logger.info("--> test state from the current master is accepted");
previousState = state;
ZenDiscovery.validateIncomingState(logger, ClusterState.builder(previousState)
.nodes(DiscoveryNodes.builder(previousState.nodes()).masterNodeId("master")).incrementVersion().build(), previousState);
logger.info("--> testing rejection of another cluster name");
try {
ZenDiscovery.validateIncomingState(logger, ClusterState.builder(new ClusterName(randomAlphaOfLength(10)))
.nodes(previousState.nodes()).build(), previousState);
fail("node accepted state with another cluster name");
} catch (IllegalStateException OK) {
assertThat(OK.toString(), containsString("received state from a node that is not part of the cluster"));
}
logger.info("--> testing rejection of a cluster state with wrong local node");
try {
state = ClusterState.builder(previousState)
.nodes(DiscoveryNodes.builder(previousState.nodes()).localNodeId("_non_existing_").build())
.incrementVersion().build();
ZenDiscovery.validateIncomingState(logger, state, previousState);
fail("node accepted state with non-existence local node");
} catch (IllegalStateException OK) {
assertThat(OK.toString(), containsString("received state with a local node that does not match the current local node"));
}
try {
DiscoveryNode otherNode = new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
state = ClusterState.builder(previousState).nodes(
DiscoveryNodes.builder(previousState.nodes()).add(otherNode)
.localNodeId(otherNode.getId()).build()
).incrementVersion().build();
ZenDiscovery.validateIncomingState(logger, state, previousState);
fail("node accepted state with existent but wrong local node");
} catch (IllegalStateException OK) {
assertThat(OK.toString(), containsString("received state with a local node that does not match the current local node"));
}
logger.info("--> testing acceptance of an old cluster state");
final ClusterState incomingState = previousState;
previousState = ClusterState.builder(previousState).incrementVersion().build();
final ClusterState finalPreviousState = previousState;
final IllegalStateException e =
expectThrows(IllegalStateException.class, () -> ZenDiscovery.validateIncomingState(logger, incomingState, finalPreviousState));
final String message = String.format(
Locale.ROOT,
"rejecting cluster state version [%d] uuid [%s] received from [%s]",
incomingState.version(),
incomingState.stateUUID(),
incomingState.nodes().getMasterNodeId()
);
assertThat(e, hasToString("java.lang.IllegalStateException: " + message));
ClusterState higherVersionState = ClusterState.builder(previousState).incrementVersion().build();
// remove the master of the node (but still have a previous cluster state with it)!
higherVersionState = ClusterState.builder(higherVersionState)
.nodes(DiscoveryNodes.builder(higherVersionState.nodes()).masterNodeId(null)).build();
// an older version from a *new* master is also OK!
state = ClusterState.builder(previousState)
.nodes(DiscoveryNodes.builder(previousState.nodes()).masterNodeId("_new_master_").build())
.build();
ZenDiscovery.validateIncomingState(logger, state, higherVersionState);
}
}

4
core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
View File

@ -70,7 +70,6 @@ import org.elasticsearch.indices.mapper.MapperRegistry;
import org.elasticsearch.script.ScriptContextRegistry;
import org.elasticsearch.script.ScriptEngineRegistry;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptSettings;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.test.ClusterServiceUtils;
import org.elasticsearch.test.ESTestCase;
@ -130,8 +129,7 @@ public class IndexModuleTests extends ESTestCase {
bigArrays = new BigArrays(settings, circuitBreakerService);
ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(emptyList());
ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
scriptService = new ScriptService(settings, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
scriptService = new ScriptService(settings, scriptEngineRegistry, scriptContextRegistry);
clusterService = ClusterServiceUtils.createClusterService(threadPool);
nodeEnvironment = new NodeEnvironment(settings, environment);
mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();

31
core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
View File

@ -23,7 +23,6 @@ import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.test.ESTestCase;
@ -32,21 +31,14 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.hamcrest.Matchers.containsString;
public class ScriptContextTests extends ESTestCase {
private static final String PLUGIN_NAME = "testplugin";
private static final String SCRIPT_PLUGIN_CUSTOM_SETTING = "script." + PLUGIN_NAME + "_custom_globally_disabled_op";
private static final String SCRIPT_ENGINE_CUSTOM_SETTING = "script.engine." + MockScriptEngine.NAME + ".inline." + PLUGIN_NAME + "_custom_exp_disabled_op";
private ScriptSettings scriptSettings;
ScriptService makeScriptService() throws Exception {
Settings settings = Settings.builder()
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
.put(SCRIPT_PLUGIN_CUSTOM_SETTING, "false")
.put(SCRIPT_ENGINE_CUSTOM_SETTING, "false")
.put("script.contexts_allowed", "search, aggs, testplugin_custom_op")
.build();
MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap("1", script -> "1"));
@ -56,8 +48,7 @@ public class ScriptContextTests extends ESTestCase {
new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"),
new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op"));
ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
ScriptService scriptService = new ScriptService(settings, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
ScriptService scriptService = new ScriptService(settings, scriptEngineRegistry, scriptContextRegistry);
ClusterState empty = ClusterState.builder(new ClusterName("_name")).build();
ScriptMetaData smd = empty.metaData().custom(ScriptMetaData.TYPE);
@ -69,8 +60,6 @@ public class ScriptContextTests extends ESTestCase {
return scriptService;
}
public void testCustomGlobalScriptContextSettings() throws Exception {
ScriptService scriptService = makeScriptService();
for (ScriptType scriptType : ScriptType.values()) {
@ -78,12 +67,10 @@ public class ScriptContextTests extends ESTestCase {
Script script = new Script(scriptType, MockScriptEngine.NAME, "1", Collections.emptyMap());
scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op"));
fail("script compilation should have been rejected");
} catch (IllegalStateException e) {
assertThat(e.getMessage(), containsString("scripts of type [" + scriptType + "], operation [" + PLUGIN_NAME + "_custom_globally_disabled_op] and lang [" + MockScriptEngine.NAME + "] are disabled"));
} catch (IllegalArgumentException e) {
assertTrue(e.getMessage(), e.getMessage().contains("cannot execute scripts using [" + PLUGIN_NAME + "_custom_globally_disabled_op] context"));
}
}
assertSettingDeprecationsAndWarnings(
ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, SCRIPT_PLUGIN_CUSTOM_SETTING, SCRIPT_ENGINE_CUSTOM_SETTING));
}
public void testCustomScriptContextSettings() throws Exception {
@ -92,16 +79,14 @@ public class ScriptContextTests extends ESTestCase {
try {
scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"));
fail("script compilation should have been rejected");
} catch (IllegalStateException e) {
assertTrue(e.getMessage(), e.getMessage().contains("scripts of type [inline], operation [" + PLUGIN_NAME + "_custom_exp_disabled_op] and lang [" + MockScriptEngine.NAME + "] are disabled"));
} catch (IllegalArgumentException e) {
assertTrue(e.getMessage(), e.getMessage().contains("cannot execute scripts using [" + PLUGIN_NAME + "_custom_exp_disabled_op] context"));
}
// still works for other script contexts
assertNotNull(scriptService.compile(script, ScriptContext.Standard.AGGS));
assertNotNull(scriptService.compile(script, ScriptContext.Standard.SEARCH));
assertNotNull(scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_op")));
assertSettingDeprecationsAndWarnings(
ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, SCRIPT_PLUGIN_CUSTOM_SETTING, SCRIPT_ENGINE_CUSTOM_SETTING));
}
public void testUnknownPluginScriptContext() throws Exception {
@ -115,8 +100,6 @@ public class ScriptContextTests extends ESTestCase {
assertTrue(e.getMessage(), e.getMessage().contains("script context [" + PLUGIN_NAME + "_unknown] not supported"));
}
}
assertSettingDeprecationsAndWarnings(
ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, SCRIPT_PLUGIN_CUSTOM_SETTING, SCRIPT_ENGINE_CUSTOM_SETTING));
}
public void testUnknownCustomScriptContext() throws Exception {
@ -136,7 +119,5 @@ public class ScriptContextTests extends ESTestCase {
assertTrue(e.getMessage(), e.getMessage().contains("script context [test] not supported"));
}
}
assertSettingDeprecationsAndWarnings(
ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, SCRIPT_PLUGIN_CUSTOM_SETTING, SCRIPT_ENGINE_CUSTOM_SETTING));
}
}

254
core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
View File

@ -1,254 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.search.lookup.SearchLookup;
import org.elasticsearch.test.ESTestCase;
import org.junit.After;
import org.junit.Before;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static java.util.Collections.unmodifiableMap;
import static org.elasticsearch.common.util.set.Sets.newHashSet;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.notNullValue;
import static org.hamcrest.Matchers.containsString;
// TODO: this needs to be a base test class, and all scripting engines extend it
public class ScriptModesTests extends ESTestCase {
ScriptSettings scriptSettings;
ScriptContextRegistry scriptContextRegistry;
private ScriptContext[] scriptContexts;
private Map<String, ScriptEngine> scriptEngines;
private ScriptModes scriptModes;
private Set<String> checkedSettings;
private boolean assertAllSettingsWereChecked;
private boolean assertScriptModesNonNull;
@Before
public void setupScriptEngines() {
//randomly register custom script contexts
int randomInt = randomIntBetween(0, 3);
//prevent duplicates using map
Map<String, ScriptContext.Plugin> contexts = new HashMap<>();
for (int i = 0; i < randomInt; i++) {
String plugin = randomAlphaOfLength(randomIntBetween(1, 10));
String operation = randomAlphaOfLength(randomIntBetween(1, 30));
String context = plugin + "-" + operation;
contexts.put(context, new ScriptContext.Plugin(plugin, operation));
}
scriptContextRegistry = new ScriptContextRegistry(contexts.values());
scriptContexts = scriptContextRegistry.scriptContexts().toArray(new ScriptContext[scriptContextRegistry.scriptContexts().size()]);
scriptEngines = buildScriptEnginesByLangMap(newHashSet(new CustomScriptEngine()));
ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(scriptEngines.values());
scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
checkedSettings = new HashSet<>();
assertAllSettingsWereChecked = true;
assertScriptModesNonNull = true;
}
@After
public void assertAllSettingsWereChecked() {
if (assertScriptModesNonNull) {
assertThat(scriptModes, notNullValue());
int numberOfSettings = ScriptType.values().length * scriptContextRegistry.scriptContexts().size();
numberOfSettings += 2; // for top-level inline/store settings
assertThat(scriptModes.scriptEnabled.size(), equalTo(numberOfSettings));
if (assertAllSettingsWereChecked) {
assertThat(checkedSettings.size(), equalTo(numberOfSettings));
}
}
}
public void testDefaultSettings() {
this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, Settings.EMPTY);
assertScriptModesAllOps(false, ScriptType.STORED, ScriptType.INLINE);
}
public void testMissingSetting() {
assertAllSettingsWereChecked = false;
this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, Settings.EMPTY);
try {
scriptModes.getScriptEnabled("non_existing", randomFrom(ScriptType.values()), randomFrom(scriptContexts));
fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), containsString("not found for lang [non_existing]"));
}
}
public void testScriptTypeGenericSettings() {
int randomInt = randomIntBetween(1, ScriptType.values().length - 1);
Set<ScriptType> randomScriptTypesSet = new HashSet<>();
boolean[] randomScriptModes = new boolean[randomInt];
for (int i = 0; i < randomInt; i++) {
boolean added = false;
while (added == false) {
added = randomScriptTypesSet.add(randomFrom(ScriptType.values()));
}
randomScriptModes[i] = randomBoolean();
}
ScriptType[] randomScriptTypes = randomScriptTypesSet.toArray(new ScriptType[randomScriptTypesSet.size()]);
List<String> deprecated = new ArrayList<>();
Settings.Builder builder = Settings.builder();
for (int i = 0; i < randomInt; i++) {
builder.put("script" + "." + randomScriptTypes[i].getName(), randomScriptModes[i]);
deprecated.add("script" + "." + randomScriptTypes[i].getName());
}
this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, builder.build());
for (int i = 0; i < randomInt; i++) {
assertScriptModesAllOps(randomScriptModes[i], randomScriptTypes[i]);
}
if (randomScriptTypesSet.contains(ScriptType.STORED) == false) {
assertScriptModesAllOps(false, ScriptType.STORED);
}
if (randomScriptTypesSet.contains(ScriptType.INLINE) == false) {
assertScriptModesAllOps(false, ScriptType.INLINE);
}
assertSettingDeprecationsAndWarnings(
ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, deprecated.toArray(new String[] {})));
}
public void testScriptContextGenericSettings() {
int randomInt = randomIntBetween(1, scriptContexts.length - 1);
Set<ScriptContext> randomScriptContextsSet = new HashSet<>();
boolean[] randomScriptModes = new boolean[randomInt];
for (int i = 0; i < randomInt; i++) {
boolean added = false;
while (added == false) {
added = randomScriptContextsSet.add(randomFrom(scriptContexts));
}
randomScriptModes[i] = randomBoolean();
}
ScriptContext[] randomScriptContexts = randomScriptContextsSet.toArray(new ScriptContext[randomScriptContextsSet.size()]);
List<String> deprecated = new ArrayList<>();
Settings.Builder builder = Settings.builder();
for (int i = 0; i < randomInt; i++) {
builder.put("script" + "." + randomScriptContexts[i].getKey(), randomScriptModes[i]);
deprecated.add("script" + "." + randomScriptContexts[i].getKey());
}
this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, builder.build());
for (int i = 0; i < randomInt; i++) {
assertScriptModesAllTypes(randomScriptModes[i], randomScriptContexts[i]);
}
ScriptContext[] complementOf = complementOf(randomScriptContexts);
assertScriptModes(false, new ScriptType[]{ScriptType.STORED, ScriptType.INLINE}, complementOf);
assertSettingDeprecationsAndWarnings(
ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, deprecated.toArray(new String[] {})));
}
public void testConflictingScriptTypeAndOpGenericSettings() {
ScriptContext scriptContext = randomFrom(scriptContexts);
Settings.Builder builder = Settings.builder()
.put("script." + scriptContext.getKey(), "false")
.put("script.stored", "true")
.put("script.inline", "true");
//operations generic settings have precedence over script type generic settings
this.scriptModes = new ScriptModes(scriptContextRegistry, scriptSettings, builder.build());
assertScriptModesAllTypes(false, scriptContext);
ScriptContext[] complementOf = complementOf(scriptContext);
assertScriptModes(true, new ScriptType[]{ScriptType.STORED}, complementOf);
assertScriptModes(true, new ScriptType[]{ScriptType.INLINE}, complementOf);
assertSettingDeprecationsAndWarnings(
ScriptSettingsTests.buildDeprecatedSettingsArray(
scriptSettings, "script." + scriptContext.getKey(), "script.stored", "script.inline"));
}
private void assertScriptModesAllOps(boolean expectedScriptEnabled, ScriptType... scriptTypes) {
assertScriptModes(expectedScriptEnabled, scriptTypes, scriptContexts);
}
private void assertScriptModesAllTypes(boolean expectedScriptEnabled, ScriptContext... scriptContexts) {
assertScriptModes(expectedScriptEnabled, ScriptType.values(), scriptContexts);
}
private void assertScriptModes(boolean expectedScriptEnabled, ScriptType[] scriptTypes, ScriptContext... scriptContexts) {
assert scriptTypes.length > 0;
assert scriptContexts.length > 0;
for (ScriptType scriptType : scriptTypes) {
checkedSettings.add("script.engine.custom." + scriptType);
for (ScriptContext scriptContext : scriptContexts) {
assertThat("custom." + scriptType + "." + scriptContext.getKey() + " doesn't have the expected value",
scriptModes.getScriptEnabled("custom", scriptType, scriptContext), equalTo(expectedScriptEnabled));
checkedSettings.add("custom." + scriptType + "." + scriptContext);
}
}
}
private ScriptContext[] complementOf(ScriptContext... scriptContexts) {
Map<String, ScriptContext> copy = new HashMap<>();
for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) {
copy.put(scriptContext.getKey(), scriptContext);
}
for (ScriptContext scriptContext : scriptContexts) {
copy.remove(scriptContext.getKey());
}
return copy.values().toArray(new ScriptContext[copy.size()]);
}
static Map<String, ScriptEngine> buildScriptEnginesByLangMap(Set<ScriptEngine> scriptEngines) {
Map<String, ScriptEngine> builder = new HashMap<>();
for (ScriptEngine scriptEngine : scriptEngines) {
String type = scriptEngine.getType();
builder.put(type, scriptEngine);
}
return unmodifiableMap(builder);
}
private static class CustomScriptEngine implements ScriptEngine {
public static final String NAME = "custom";
@Override
public String getType() {
return NAME;
}
@Override
public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
return null;
}
@Override
public ExecutableScript executable(CompiledScript compiledScript, @Nullable Map<String, Object> vars) {
return null;
}
@Override
public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map<String, Object> vars) {
return null;
}
@Override
public void close() {
}
}
}

131
core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
View File

@ -57,7 +57,6 @@ public class ScriptServiceTests extends ESTestCase {
private Map<String, ScriptEngine> scriptEnginesByLangMap;
private ScriptEngineRegistry scriptEngineRegistry;
private ScriptContextRegistry scriptContextRegistry;
private ScriptSettings scriptSettings;
private ScriptContext[] scriptContexts;
private ScriptService scriptService;
private Settings baseSettings;
@ -80,8 +79,6 @@ public class ScriptServiceTests extends ESTestCase {
scriptEngine = new TestEngine();
dangerousScriptEngine = new TestDangerousEngine();
TestEngine defaultScriptServiceEngine = new TestEngine(Script.DEFAULT_SCRIPT_LANG) {};
scriptEnginesByLangMap = ScriptModesTests.buildScriptEnginesByLangMap(
new HashSet<>(Arrays.asList(scriptEngine, defaultScriptServiceEngine)));
//randomly register custom script contexts
int randomInt = randomIntBetween(0, 3);
//prevent duplicates using map
@ -101,15 +98,13 @@ public class ScriptServiceTests extends ESTestCase {
scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(scriptEngine, dangerousScriptEngine,
defaultScriptServiceEngine));
scriptContextRegistry = new ScriptContextRegistry(contexts.values());
scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
scriptContexts = scriptContextRegistry.scriptContexts().toArray(new ScriptContext[scriptContextRegistry.scriptContexts().size()]);
logger.info("--> setup script service");
}
private void buildScriptService(Settings additionalSettings) throws IOException {
Settings finalSettings = Settings.builder().put(baseSettings).put(additionalSettings).build();
// TODO:
scriptService = new ScriptService(finalSettings, scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
scriptService = new ScriptService(finalSettings, scriptEngineRegistry, scriptContextRegistry) {
@Override
StoredScriptSource getScriptFromClusterState(String id, String lang) {
//mock the script that gets retrieved from an index
@ -179,33 +174,25 @@ public class ScriptServiceTests extends ESTestCase {
public void testAllowSomeScriptTypeSettings() throws IOException {
Settings.Builder builder = Settings.builder();
builder.put("script.types_allowed", "inline");
builder.put("script.engine.painless.stored", false);
buildScriptService(builder.build());
assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.Standard.SEARCH);
assertCompileRejected("painless", "script", ScriptType.STORED, ScriptContext.Standard.SEARCH);
assertSettingDeprecationsAndWarnings(
ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, "script.engine.painless.stored"));
}
public void testAllowSomeScriptContextSettings() throws IOException {
Settings.Builder builder = Settings.builder();
builder.put("script.contexts_allowed", "search, aggs");
builder.put("script.update", false);
buildScriptService(builder.build());
assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.Standard.SEARCH);
assertCompileAccepted("painless", "script", ScriptType.INLINE, ScriptContext.Standard.AGGS);
assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.Standard.UPDATE);
assertSettingDeprecationsAndWarnings(
ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, "script.update"));
}
public void testAllowNoScriptTypeSettings() throws IOException {
Settings.Builder builder = Settings.builder();
builder.put("script.types_allowed", "");
builder.put("script.types_allowed", "none");
buildScriptService(builder.build());
assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.Standard.SEARCH);
@ -214,7 +201,7 @@ public class ScriptServiceTests extends ESTestCase {
public void testAllowNoScriptContextSettings() throws IOException {
Settings.Builder builder = Settings.builder();
builder.put("script.contexts_allowed", "");
builder.put("script.contexts_allowed", "none");
buildScriptService(builder.build());
assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.Standard.SEARCH);
@ -223,109 +210,6 @@ public class ScriptServiceTests extends ESTestCase {
assertCompileRejected("painless", "script", ScriptType.INLINE, ScriptContext.Standard.INGEST);
}
public void testDefaultBehaviourFineGrainedSettings() throws IOException {
Settings.Builder builder = Settings.builder();
buildScriptService(builder.build());
for (ScriptContext scriptContext : scriptContexts) {
assertCompileRejected("dtest", "script", ScriptType.INLINE, scriptContext);
assertCompileRejected("dtest", "script", ScriptType.STORED, scriptContext);
}
}
public void testFineGrainedSettings() throws IOException {
//collect the fine-grained settings to set for this run
int numScriptSettings = randomIntBetween(0, ScriptType.values().length);
Map<ScriptType, Boolean> scriptSourceSettings = new HashMap<>();
for (int i = 0; i < numScriptSettings; i++) {
ScriptType scriptType;
do {
scriptType = randomFrom(ScriptType.values());
} while (scriptSourceSettings.containsKey(scriptType));
scriptSourceSettings.put(scriptType, randomBoolean());
}
int numScriptContextSettings = randomIntBetween(0, this.scriptContextRegistry.scriptContexts().size());
Map<ScriptContext, Boolean> scriptContextSettings = new HashMap<>();
for (int i = 0; i < numScriptContextSettings; i++) {
ScriptContext scriptContext;
do {
scriptContext = randomFrom(this.scriptContexts);
} while (scriptContextSettings.containsKey(scriptContext));
scriptContextSettings.put(scriptContext, randomBoolean());
}
int numEngineSettings = randomIntBetween(0, ScriptType.values().length * scriptContexts.length);
Map<String, Boolean> engineSettings = new HashMap<>();
for (int i = 0; i < numEngineSettings; i++) {
String settingKey;
do {
ScriptType scriptType = randomFrom(ScriptType.values());
ScriptContext scriptContext = randomFrom(this.scriptContexts);
settingKey = scriptEngine.getType() + "." + scriptType + "." + scriptContext.getKey();
} while (engineSettings.containsKey(settingKey));
engineSettings.put(settingKey, randomBoolean());
}
List<String> deprecated = new ArrayList<>();
//set the selected fine-grained settings
Settings.Builder builder = Settings.builder();
for (Map.Entry<ScriptType, Boolean> entry : scriptSourceSettings.entrySet()) {
if (entry.getValue()) {
builder.put("script" + "." + entry.getKey().getName(), "true");
} else {
builder.put("script" + "." + entry.getKey().getName(), "false");
}
deprecated.add("script" + "." + entry.getKey().getName());
}
for (Map.Entry<ScriptContext, Boolean> entry : scriptContextSettings.entrySet()) {
if (entry.getValue()) {
builder.put("script" + "." + entry.getKey().getKey(), "true");
} else {
builder.put("script" + "." + entry.getKey().getKey(), "false");
}
deprecated.add("script" + "." + entry.getKey().getKey());
}
for (Map.Entry<String, Boolean> entry : engineSettings.entrySet()) {
int delimiter = entry.getKey().indexOf('.');
String part1 = entry.getKey().substring(0, delimiter);
String part2 = entry.getKey().substring(delimiter + 1);
String lang = randomFrom(scriptEnginesByLangMap.get(part1).getType());
if (entry.getValue()) {
builder.put("script.engine" + "." + lang + "." + part2, "true");
} else {
builder.put("script.engine" + "." + lang + "." + part2, "false");
}
deprecated.add("script.engine" + "." + lang + "." + part2);
}
buildScriptService(builder.build());
for (ScriptType scriptType : ScriptType.values()) {
String script = "script";
for (ScriptContext scriptContext : this.scriptContexts) {
//fallback mechanism: 1) engine specific settings 2) op based settings 3) source based settings
Boolean scriptEnabled = engineSettings.get(dangerousScriptEngine.getType() + "." + scriptType + "." + scriptContext.getKey());
if (scriptEnabled == null) {
scriptEnabled = scriptContextSettings.get(scriptContext);
}
if (scriptEnabled == null) {
scriptEnabled = scriptSourceSettings.get(scriptType);
}
if (scriptEnabled == null) {
scriptEnabled = DEFAULT_SCRIPT_ENABLED.get(scriptType);
}
String lang = dangerousScriptEngine.getType();
if (scriptEnabled) {
assertCompileAccepted(lang, script, scriptType, scriptContext);
} else {
assertCompileRejected(lang, script, scriptType, scriptContext);
}
}
}
assertSettingDeprecationsAndWarnings(
ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, deprecated.toArray(new String[] {})));
}
public void testCompileNonRegisteredContext() throws IOException {
buildScriptService(Settings.EMPTY);
String pluginName;
@ -378,14 +262,11 @@ public class ScriptServiceTests extends ESTestCase {
public void testCompilationStatsOnCacheHit() throws IOException {
Settings.Builder builder = Settings.builder();
builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), 1);
builder.put("script.inline", "true");
buildScriptService(builder.build());
Script script = new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap());
scriptService.compile(script, randomFrom(scriptContexts));
scriptService.compile(script, randomFrom(scriptContexts));
assertEquals(1L, scriptService.stats().getCompilations());
assertSettingDeprecationsAndWarnings(
ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, "script.inline"));
}
public void testIndexedScriptCountedInCompilationStats() throws IOException {
@ -397,25 +278,19 @@ public class ScriptServiceTests extends ESTestCase {
public void testCacheEvictionCountedInCacheEvictionsStats() throws IOException {
Settings.Builder builder = Settings.builder();
builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), 1);
builder.put("script.inline", "true");
buildScriptService(builder.build());
scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), randomFrom(scriptContexts));
scriptService.compile(new Script(ScriptType.INLINE, "test", "2+2", Collections.emptyMap()), randomFrom(scriptContexts));
assertEquals(2L, scriptService.stats().getCompilations());
assertEquals(1L, scriptService.stats().getCacheEvictions());
assertSettingDeprecationsAndWarnings(
ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, "script.inline"));
}
public void testDefaultLanguage() throws IOException {
Settings.Builder builder = Settings.builder();
builder.put("script.inline", "true");
buildScriptService(builder.build());
CompiledScript script = scriptService.compile(
new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "1 + 1", Collections.emptyMap()), randomFrom(scriptContexts));
assertEquals(script.lang(), Script.DEFAULT_SCRIPT_LANG);
assertSettingDeprecationsAndWarnings(
ScriptSettingsTests.buildDeprecatedSettingsArray(scriptSettings, "script.inline"));
}
public void testStoreScript() throws Exception {

98
core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
View File

@ -1,98 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.script;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.search.lookup.SearchLookup;
import org.elasticsearch.test.ESTestCase;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;
import static org.hamcrest.Matchers.equalTo;
public class ScriptSettingsTests extends ESTestCase {
public static Setting<?>[] buildDeprecatedSettingsArray(ScriptSettings scriptSettings, String... keys) {
Setting<?>[] settings = new Setting[keys.length];
int count = 0;
for (Setting<?> setting : scriptSettings.getSettings()) {
for (String key : keys) {
if (setting.getKey().equals(key)) {
settings[count++] = setting;
}
}
}
return settings;
}
public void testSettingsAreProperlyPropogated() {
ScriptEngineRegistry scriptEngineRegistry =
new ScriptEngineRegistry(Collections.singletonList(new CustomScriptEngine()));
ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
boolean enabled = randomBoolean();
Settings s = Settings.builder().put("script.inline", enabled).build();
for (Iterator<Setting<Boolean>> iter = scriptSettings.getScriptLanguageSettings().iterator(); iter.hasNext();) {
Setting<Boolean> setting = iter.next();
if (setting.getKey().endsWith(".inline")) {
assertThat("inline settings should have propagated", setting.get(s), equalTo(enabled));
assertThat(setting.getDefaultRaw(s), equalTo(Boolean.toString(enabled)));
}
}
assertSettingDeprecationsAndWarnings(buildDeprecatedSettingsArray(scriptSettings, "script.inline"));
}
private static class CustomScriptEngine implements ScriptEngine {
public static final String NAME = "custom";
@Override
public String getType() {
return NAME;
}
@Override
public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
return null;
}
@Override
public ExecutableScript executable(CompiledScript compiledScript, @Nullable Map<String, Object> vars) {
return null;
}
@Override
public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map<String, Object> vars) {
return null;
}
@Override
public void close() {
}
}
}

56
core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
View File

@ -1162,7 +1162,61 @@ public class DateHistogramIT extends ESIntegTestCase {
assertThat(bucket.getDocCount(), equalTo(0L));
}
}
internalCluster().wipeIndices("test12278");
internalCluster().wipeIndices(index);
}
/**
* Test date histogram aggregation with day interval, offset and
* extended bounds (see https://github.com/elastic/elasticsearch/issues/23776)
*/
public void testSingleValueFieldWithExtendedBoundsOffset() throws Exception {
String index = "test23776";
prepareCreate(index)
.setSettings(Settings.builder().put(indexSettings()).put("index.number_of_shards", 1).put("index.number_of_replicas", 0))
.execute().actionGet();
List<IndexRequestBuilder> builders = new ArrayList<>();
builders.add(indexDoc(index, DateTime.parse("2016-01-03T08:00:00.000Z"), 1));
builders.add(indexDoc(index, DateTime.parse("2016-01-03T08:00:00.000Z"), 2));
builders.add(indexDoc(index, DateTime.parse("2016-01-06T08:00:00.000Z"), 3));
builders.add(indexDoc(index, DateTime.parse("2016-01-06T08:00:00.000Z"), 4));
indexRandom(true, builders);
ensureSearchable(index);
SearchResponse response = null;
// retrieve those docs with the same time zone and extended bounds
response = client()
.prepareSearch(index)
.addAggregation(
dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.days(1)).offset("+6h").minDocCount(0)
.extendedBounds(new ExtendedBounds("2016-01-01T06:00:00Z", "2016-01-08T08:00:00Z"))
).execute().actionGet();
assertSearchResponse(response);
Histogram histo = response.getAggregations().get("histo");
assertThat(histo, notNullValue());
assertThat(histo.getName(), equalTo("histo"));
List<? extends Bucket> buckets = histo.getBuckets();
assertThat(buckets.size(), equalTo(8));
assertEquals("2016-01-01T06:00:00.000Z", buckets.get(0).getKeyAsString());
assertEquals(0, buckets.get(0).getDocCount());
assertEquals("2016-01-02T06:00:00.000Z", buckets.get(1).getKeyAsString());
assertEquals(0, buckets.get(1).getDocCount());
assertEquals("2016-01-03T06:00:00.000Z", buckets.get(2).getKeyAsString());
assertEquals(2, buckets.get(2).getDocCount());
assertEquals("2016-01-04T06:00:00.000Z", buckets.get(3).getKeyAsString());
assertEquals(0, buckets.get(3).getDocCount());
assertEquals("2016-01-05T06:00:00.000Z", buckets.get(4).getKeyAsString());
assertEquals(0, buckets.get(4).getDocCount());
assertEquals("2016-01-06T06:00:00.000Z", buckets.get(5).getKeyAsString());
assertEquals(2, buckets.get(5).getDocCount());
assertEquals("2016-01-07T06:00:00.000Z", buckets.get(6).getKeyAsString());
assertEquals(0, buckets.get(6).getDocCount());
assertEquals("2016-01-08T06:00:00.000Z", buckets.get(7).getKeyAsString());
assertEquals(0, buckets.get(7).getDocCount());
internalCluster().wipeIndices(index);
}
public void testSingleValueWithMultipleDateFormatsFromMapping() throws Exception {

5
core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java
View File

@ -29,7 +29,6 @@ import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptContextRegistry;
import org.elasticsearch.script.ScriptEngineRegistry;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptSettings;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.aggregations.ParsedAggregation;
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@ -122,10 +121,8 @@ public class InternalScriptedMetricTests extends InternalAggregationTestCase<Int
}));
ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(scriptEngine));
ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
try {
return new ScriptService(Settings.EMPTY, scriptEngineRegistry, scriptContextRegistry,
scriptSettings);
return new ScriptService(Settings.EMPTY, scriptEngineRegistry, scriptContextRegistry);
} catch (IOException e) {
throw new ElasticsearchException(e);
}

4
core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java
View File

@ -38,7 +38,6 @@ import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptContextRegistry;
import org.elasticsearch.script.ScriptEngineRegistry;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptSettings;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.aggregations.AggregatorTestCase;
import org.junit.BeforeClass;
@ -201,10 +200,9 @@ public class ScriptedMetricAggregatorTests extends AggregatorTestCase {
MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, SCRIPTS);
ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(scriptEngine));
ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
ScriptService scriptService;
try {
scriptService = new ScriptService(Settings.EMPTY, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
scriptService = new ScriptService(Settings.EMPTY, scriptEngineRegistry, scriptContextRegistry);
} catch (IOException e) {
throw new ElasticsearchException(e);
}

5
core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
View File

@ -57,7 +57,6 @@ import org.elasticsearch.script.ScriptContextRegistry;
import org.elasticsearch.script.ScriptEngineRegistry;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptServiceTests.TestEngine;
import org.elasticsearch.script.ScriptSettings;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.SearchModule;
@ -89,11 +88,9 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.put(Environment.PATH_CONF_SETTING.getKey(), genericConfigFolder)
.build();
Environment environment = new Environment(baseSettings);
ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new TestEngine()));
ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
scriptService = new ScriptService(baseSettings, scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
scriptService = new ScriptService(baseSettings, scriptEngineRegistry, scriptContextRegistry) {
@Override
public CompiledScript compile(Script script, ScriptContext scriptContext) {
return new CompiledScript(ScriptType.INLINE, "mockName", "test", script);

2
dev-tools/smoke_test_rc.py
View File

@ -200,7 +200,7 @@ def smoke_test_release(release, files, hash, plugins):
headers = {}
print(' Starting elasticsearch deamon from [%s]' % es_dir)
try:
run('%s; %s -Enode.name=smoke_tester -Ecluster.name=prepare_release -Escript.inline=true -Escript.stored=true -Erepositories.url.allowed_urls=http://snapshot.test* %s -Epidfile=%s -Enode.portsfile=true'
run('%s; %s -Enode.name=smoke_tester -Ecluster.name=prepare_release -Erepositories.url.allowed_urls=http://snapshot.test* %s -Epidfile=%s -Enode.portsfile=true'
% (java_exe(), es_run_path, '-d', os.path.join(es_dir, 'es-smoke.pid')))
if not wait_for_node_startup(es_dir, header=headers):
print("elasticsearch logs:")

1
distribution/src/main/packaging/scripts/postinst
View File

@ -110,7 +110,6 @@ chown -R elasticsearch:elasticsearch /var/lib/elasticsearch
chown -R elasticsearch:elasticsearch /var/log/elasticsearch
chown -R root:elasticsearch /etc/elasticsearch
chmod 0750 /etc/elasticsearch
chmod 0750 /etc/elasticsearch/scripts
if [ -f /etc/sysconfig/elasticsearch ]; then
chmod 0660 /etc/sysconfig/elasticsearch

6
distribution/src/main/packaging/scripts/prerm
View File

@ -79,12 +79,6 @@ if [ "$REMOVE_SERVICE" = "true" ]; then
if command -v update-rc.d >/dev/null; then
update-rc.d elasticsearch remove >/dev/null || true
fi
SCRIPTS_DIR="/etc/elasticsearch/scripts"
# delete the scripts directory if and only if empty
if [ -d "$SCRIPTS_DIR" ]; then
rmdir --ignore-fail-on-non-empty "$SCRIPTS_DIR"
fi
fi

2
docs/build.gradle
View File

@ -57,8 +57,6 @@ buildRestTests.expectedUnconvertedCandidates = [
]
integTestCluster {
setting 'script.inline', 'true'
setting 'script.stored', 'true'
setting 'script.max_compilations_per_minute', '1000'
/* Enable regexes in painless so our tests don't complain about example
* snippets that use them. */

5
docs/reference/migration/migrate_6_0/plugins.asciidoc
View File

@ -55,6 +55,11 @@ You must set those settings per repository instead. Respectively `account`, `con
`location_mode`, `chunk_size` and `compress`.
See {plugins}/repository-azure-usage.html#repository-azure-repository-settings[Azure Repository settings].
==== GCS Repository plugin
* The `service_account` setting has been removed. A service account json credential file must now be
specified in the <<secure-settings, elasticsearch keystore>>.
==== EC2 Discovery plugin
* Specifying ec2 signer type has been removed, including `cloud.aws.signer` and `cloud.aws.ec2.signer`.

2
docs/reference/migration/migrate_6_0/scripting.asciidoc
View File

@ -24,5 +24,5 @@ The `_index` variable has been removed. If you used it for advanced scoring, con
==== Script Settings
All of the existing scripting security settings have been deprecated. Instead
All of the existing scripting security settings have been removed. Instead
they are replaced with `script.allowed_types` and `script.allowed_contexts`.

2
docs/reference/migration/migrate_6_0/settings.asciidoc
View File

@ -74,5 +74,5 @@ deprecation warning.
==== Script Settings
All of the existing scripting security settings have been deprecated. Instead
All of the existing scripting security settings have been removed. Instead
they are replaced with `script.allowed_types` and `script.allowed_contexts`.

108
docs/reference/modules/scripting/security.asciidoc
View File

@ -93,7 +93,8 @@ security of the Elasticsearch deployment.
By default all script types are allowed to be executed. This can be modified using the
setting `script.allowed_types`. Only the types specified as part of the setting will be
allowed to be executed.
allowed to be executed. To specify no types are allowed, set `scripts.allowed_types` to
be `none`.
[source,yaml]
----
@ -108,7 +109,8 @@ script.allowed_types: inline <1>
By default all script contexts are allowed to be executed. This can be modified using the
setting `script.allowed_contexts`. Only the contexts specified as part of the setting will
be allowed to be executed.
be allowed to be executed. To specify no contexts are allowed, set `scripts.allowed_contexts`
to be `none`.
[source,yaml]
----
@ -116,105 +118,3 @@ script.allowed_contexts: search, update <1>
----
<1> This will allow only search and update scripts to be executed but not
aggs or plugin scripts (or any other contexts).
[[deprecated-script=settings]]
[float]
=== Deprecated script settings
The following settings have all been deprecated and will be removed in 6.0:
* <<security-script-source>>
* <<security-script-context>>
* <<security-script-fine>>
Use the following instead:
* <<allowed-script-types-setting>>
* <<allowed-script-contexts-setting>>
[[security-script-source]]
[float]
=== Script source settings
Which scripts Elasticsearch will execute where is controlled by settings
starting with `scripts.`. The simplest settings allow scripts to be enabled
or disabled based on where they are stored. For example:
[source,yaml]
-----------------------------------
script.inline: false <1>
script.stored: false <2>
-----------------------------------
<1> Refuse to run scripts provided inline in the API.
<2> Refuse to run scripts stored using the API.
NOTE: These settings override the defaults mentioned
<<modules-scripting-security-do-no-weaken, above>>. Recreating the defaults
requires more fine grained settings described <<security-script-fine, below>>.
[[security-script-context]]
[float]
=== Script context settings
Scripting may also be enabled or disabled in different contexts in the
Elasticsearch API. The supported contexts are:
[horizontal]
`aggs`:: Aggregations
`search`:: Search api, Percolator API and Suggester API
`update`:: Update api
`plugin`:: Any plugin that makes use of scripts under the generic `plugin` category
Plugins can also define custom operations that they use scripts for instead
of using the generic `plugin` category. Those operations can be referred to
in the following form: `${pluginName}_${operation}`.
The following example disables scripting for `update` and `plugin` operations,
regardless of the script source or language. Scripts can still be executed
as part of `aggregations`, `search` and plugins execution though, as the above
defaults still get applied.
[source,yaml]
-----------------------------------
script.update: false
script.plugin: false
-----------------------------------
[[security-script-fine]]
[float]
=== Fine-grained script settings
First, the high-level script settings described above are applied in order
(context settings have precedence over source settings). Then fine-grained
settings which include the script language take precedence over any high-level
settings. They have two forms:
[source,yaml]
------------------------
script.engine.{lang}.{inline|stored}.{context}: true|false
------------------------
And
[source,yaml]
------------------------
script.engine.{lang}.{inline|stored}: true|false
------------------------
For example:
[source,yaml]
-----------------------------------
script.inline: false <1>
script.stored: false <1>
script.engine.painless.inline: true <2>
script.engine.painless.stored.search: true <3>
script.engine.painless.stored.aggs: true <3>
script.engine.mustache.stored.search: true <4>
-----------------------------------
<1> Disable all scripting from any source.
<2> Allow inline Painless scripts for all operations.
<3> Allow stored Painless scripts to be used for search and aggregations.
<4> Allow stored Mustache templates to be used for search.

4
docs/reference/search/search-template.asciidoc
View File

@ -28,8 +28,8 @@ documentation of the mustache project].
NOTE: The mustache language is implemented in elasticsearch as a sandboxed
scripting language, hence it obeys settings that may be used to enable or
disable scripts per language, source and operation as described in the
<<security-script-source, scripting docs>>
disable scripts per type and context as described in the
<<allowed-script-types-setting, scripting docs>>
[float]
==== More template examples

7
modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java
View File

@ -40,8 +40,7 @@ public class StoredExpressionTests extends ESIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal));
builder.put("script.engine.expression.stored.update", "false");
builder.put("script.engine.expression.stored.search", "false");
builder.put("script.contexts_allowed", "update");
return builder.build();
}
@ -72,7 +71,7 @@ public class StoredExpressionTests extends ESIntegTestCase {
.setIndices("test").setTypes("scriptTest").get();
fail("search script should have been rejected");
} catch(Exception e) {
assertThat(e.toString(), containsString("scripts of type [stored], operation [search] and lang [expression] are disabled"));
assertThat(e.toString(), containsString("cannot execute scripts using [search] context"));
}
try {
client().prepareSearch("test")
@ -80,7 +79,7 @@ public class StoredExpressionTests extends ESIntegTestCase {
new SearchSourceBuilder().aggregation(AggregationBuilders.terms("test").script(
new Script(ScriptType.STORED, "expression", "script1", Collections.emptyMap())))).get();
} catch (Exception e) {
assertThat(e.toString(), containsString("scripts of type [stored], operation [aggs] and lang [expression] are disabled"));
assertThat(e.toString(), containsString("cannot execute scripts using [aggs] context"));
}
}
}

2
modules/lang-mustache/build.gradle
View File

@ -28,7 +28,5 @@ dependencies {
}
integTestCluster {
setting 'script.inline', 'true'
setting 'script.stored', 'true'
setting 'script.max_compilations_per_minute', '1000'
}

1
plugins/repository-azure/build.gradle
View File

@ -45,5 +45,4 @@ thirdPartyAudit.excludes = [
integTestCluster {
setting 'cloud.azure.storage.my_account_test.account', 'cloudazureresource'
setting 'cloud.azure.storage.my_account_test.key', 'abcdefgh'
setting 'script.stored', 'true'
}

5
plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
View File

@ -62,8 +62,6 @@ class GoogleCloudStorageRepository extends BlobStoreRepository {
byteSizeSetting("chunk_size", MAX_CHUNK_SIZE, MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, Property.NodeScope, Property.Dynamic);
static final Setting<String> APPLICATION_NAME =
new Setting<>("application_name", GoogleCloudStoragePlugin.NAME, Function.identity(), Property.NodeScope, Property.Dynamic);
static final Setting<String> SERVICE_ACCOUNT =
new Setting<>("service_account", "_default_", Function.identity(), Property.NodeScope, Property.Dynamic, Property.Deprecated);
static final Setting<String> CLIENT_NAME = new Setting<>("client", "default", Function.identity());
static final Setting<TimeValue> HTTP_READ_TIMEOUT =
timeSetting("http.read_timeout", NO_TIMEOUT, Property.NodeScope, Property.Dynamic);
@ -82,7 +80,6 @@ class GoogleCloudStorageRepository extends BlobStoreRepository {
String bucket = getSetting(BUCKET, metadata);
String application = getSetting(APPLICATION_NAME, metadata);
String serviceAccount = SERVICE_ACCOUNT.get(metadata.settings());
String clientName = CLIENT_NAME.get(metadata.settings());
String basePath = BASE_PATH.get(metadata.settings());
@ -115,7 +112,7 @@ class GoogleCloudStorageRepository extends BlobStoreRepository {
logger.debug("using bucket [{}], base_path [{}], chunk_size [{}], compress [{}], application [{}]",
bucket, basePath, chunkSize, compress, application);
Storage client = storageService.createClient(serviceAccount, clientName, application, connectTimeout, readTimeout);
Storage client = storageService.createClient(clientName, application, connectTimeout, readTimeout);
this.blobStore = new GoogleCloudStorageBlobStore(settings, bucket, client);
}

30
plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
View File

@ -63,14 +63,13 @@ interface GoogleCloudStorageService {
/**
* Creates a client that can be used to manage Google Cloud Storage objects.
*
* @param serviceAccount path to service account file
* @param clientName name of client settings to use from secure settings
* @param application name of the application
* @param connectTimeout connection timeout for HTTP requests
* @param readTimeout read timeout for HTTP requests
* @return a Client instance that can be used to manage objects
*/
Storage createClient(String serviceAccount, String clientName, String application,
Storage createClient(String clientName, String application,
TimeValue connectTimeout, TimeValue readTimeout) throws Exception;
/**
@ -92,10 +91,10 @@ interface GoogleCloudStorageService {
}
@Override
public Storage createClient(String serviceAccountFile, String clientName, String application,
public Storage createClient(String clientName, String application,
TimeValue connectTimeout, TimeValue readTimeout) throws Exception {
try {
GoogleCredential credential = getCredential(serviceAccountFile, clientName);
GoogleCredential credential = getCredential(clientName);
NetHttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
Storage.Builder storage = new Storage.Builder(httpTransport, JacksonFactory.getDefaultInstance(),
@ -111,25 +110,10 @@ interface GoogleCloudStorageService {
}
// pkg private for tests
GoogleCredential getCredential(String serviceAccountFile, String clientName) throws IOException {
if (DEFAULT.equalsIgnoreCase(serviceAccountFile) == false) {
deprecationLogger.deprecated("Using GCS service account file from disk is deprecated. " +
"Move the file into the elasticsearch keystore.");
Path account = environment.configFile().resolve(serviceAccountFile);
if (Files.exists(account) == false) {
throw new IllegalArgumentException("Unable to find service account file [" + serviceAccountFile
+ "] defined for repository");
}
try (InputStream is = Files.newInputStream(account)) {
GoogleCredential credential = GoogleCredential.fromStream(is);
if (credential.createScopedRequired()) {
credential = credential.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL));
}
return credential;
}
} else if (credentials.containsKey(clientName)) {
return credentials.get(clientName);
GoogleCredential getCredential(String clientName) throws IOException {
GoogleCredential cred = credentials.get(clientName);
if (cred != null) {
return cred;
}
return getDefaultCredential();
}

5
plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
View File

@ -78,9 +78,8 @@ public class GoogleCloudStorageBlobStoreRepositoryTests extends ESBlobStoreRepos
public static class MockGoogleCloudStorageService implements GoogleCloudStorageService {
@Override
public Storage createClient(String serviceAccount, String accountName, String application,
TimeValue connectTimeout, TimeValue readTimeout) throws
Exception {
public Storage createClient(String accountName, String application,
TimeValue connectTimeout, TimeValue readTimeout) throws Exception {
return storage.get();
}
}

27
plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java
View File

@ -49,30 +49,7 @@ public class GoogleCloudStorageServiceTests extends ESTestCase {
return cred;
}
};
assertSame(cred, service.getCredential("_default_", "default"));
}
public void testFileCredentialBackcompat() throws Exception {
Path home = createTempDir();
Path config = home.resolve("config");
Files.createDirectories(config);
Settings settings = Settings.builder()
.put("path.home", home).build();
Environment env = new Environment(settings);
Files.copy(getDummyCredentialStream(), config.resolve("test-cred.json"));
InternalGoogleCloudStorageService service = new InternalGoogleCloudStorageService(env, Collections.emptyMap());
GoogleCredential cred = service.getCredential("test-cred.json", "default");
assertEquals("some-project-name@appspot.gserviceaccount.com", cred.getServiceAccountId());
assertWarnings("Using GCS service account file from disk is deprecated. Move the file into the elasticsearch keystore.");
}
public void testFileCredentialMissing() throws Exception {
Environment env = new Environment(Settings.builder().put("path.home", createTempDir()).build());
InternalGoogleCloudStorageService service = new InternalGoogleCloudStorageService(env, Collections.emptyMap());
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
service.getCredential("test-cred.json", "default"));
assertThat(e.getMessage(), containsString("Unable to find service account file"));
assertWarnings("Using GCS service account file from disk is deprecated. Move the file into the elasticsearch keystore.");
assertSame(cred, service.getCredential("default"));
}
public void testClientCredential() throws Exception {
@ -80,6 +57,6 @@ public class GoogleCloudStorageServiceTests extends ESTestCase {
Map<String, GoogleCredential> credentials = Collections.singletonMap("clientname", cred);
Environment env = new Environment(Settings.builder().put("path.home", createTempDir()).build());
InternalGoogleCloudStorageService service = new InternalGoogleCloudStorageService(env, credentials);
assertSame(cred, service.getCredential("_default_", "clientname"));
assertSame(cred, service.getCredential("clientname"));
}
}

34
qa/backwards-5.0/build.gradle
View File

@ -1,34 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
/* This project runs the core REST tests against a 2 node cluster where one of the nodes has a different minor. */
integTest {
includePackaged = true
}
integTestCluster {
numNodes = 4
numBwcNodes = 2
bwcVersion = project.wireCompatVersions[-1]
setting 'logger.org.elasticsearch', 'DEBUG'
}

64
qa/mixed-cluster/build.gradle Normal file
View File

@ -0,0 +1,64 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import org.elasticsearch.gradle.test.RestIntegTestTask
import org.elasticsearch.gradle.Version
apply plugin: 'elasticsearch.standalone-test'
// This is a top level task which we will add dependencies to below.
// It is a single task that can be used to backcompat tests against all versions.
task bwcTest {
description = 'Runs backwards compatibility tests.'
group = 'verification'
}
for (Version version : wireCompatVersions) {
String baseName = "v${version}"
Task mixedClusterTest = tasks.create(name: "${baseName}#mixedClusterTest", type: RestIntegTestTask) {
mustRunAfter(precommit)
includePackaged = true
}
/* This project runs the core REST tests against a 2 node cluster where one of
the nodes has a different minor. */
Object extension = extensions.findByName("${baseName}#mixedClusterTestCluster")
configure(extensions.findByName("${baseName}#mixedClusterTestCluster")) {
distribution = 'zip'
numNodes = 4
numBwcNodes = 2
bwcVersion = project.wireCompatVersions[-1]
}
Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") {
dependsOn = [mixedClusterTest]
}
bwcTest.dependsOn(versionBwcTest)
}
test.enabled = false // no unit tests for rolling upgrades, only the rest integration test
// basic integ tests includes testing bwc against the most recent version
task integTest {
dependsOn = ["v${wireCompatVersions[-1]}#bwcTest"]
}
check.dependsOn(integTest)

0
qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/IndexingIT.java → qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java
View File

4
qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/Backwards50ClientYamlTestSuiteIT.java → qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterClientYamlTestSuiteIT.java
View File

@ -26,9 +26,9 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
@TimeoutSuite(millis = 40 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell
public class Backwards50ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
public class MixedClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
public Backwards50ClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
public MixedClusterClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}

77
qa/rolling-upgrade/build.gradle
View File

@ -18,71 +18,90 @@
*/
import org.elasticsearch.gradle.test.RestIntegTestTask
import org.elasticsearch.gradle.Version
apply plugin: 'elasticsearch.standalone-test'
task oldClusterTest(type: RestIntegTestTask) {
mustRunAfter(precommit)
// This is a top level task which we will add dependencies to below.
// It is a single task that can be used to backcompat tests against all versions.
task bwcTest {
description = 'Runs backwards compatibility tests.'
group = 'verification'
}
oldClusterTestCluster {
for (Version version : wireCompatVersions) {
String baseName = "v${version}"
Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) {
mustRunAfter(precommit)
}
Object extension = extensions.findByName("${baseName}#oldClusterTestCluster")
configure(extensions.findByName("${baseName}#oldClusterTestCluster")) {
distribution = 'zip'
bwcVersion = project.wireCompatVersions[-1] // TODO: either randomize, or make this settable with sysprop
bwcVersion = version
numBwcNodes = 2
numNodes = 2
clusterName = 'rolling-upgrade'
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
if (version.onOrAfter('5.3.0')) {
setting 'http.content_type.required', 'true'
}
}
}
oldClusterTestRunner {
Task oldClusterTestRunner = tasks.getByName("${baseName}#oldClusterTestRunner")
oldClusterTestRunner.configure {
systemProperty 'tests.rest.suite', 'old_cluster'
}
}
task mixedClusterTest(type: RestIntegTestTask) {}
Task mixedClusterTest = tasks.create(name: "${baseName}#mixedClusterTest", type: RestIntegTestTask)
mixedClusterTestCluster {
dependsOn oldClusterTestRunner, 'oldClusterTestCluster#node1.stop'
configure(extensions.findByName("${baseName}#mixedClusterTestCluster")) {
dependsOn oldClusterTestRunner, "${baseName}#oldClusterTestCluster#node1.stop"
distribution = 'zip'
clusterName = 'rolling-upgrade'
unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
dataDir = "${-> oldClusterTest.nodes[1].dataDir}"
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
}
}
mixedClusterTestRunner {
Task mixedClusterTestRunner = tasks.getByName("${baseName}#mixedClusterTestRunner")
mixedClusterTestRunner.configure {
systemProperty 'tests.rest.suite', 'mixed_cluster'
finalizedBy 'oldClusterTestCluster#node0.stop'
}
finalizedBy "${baseName}#oldClusterTestCluster#node0.stop"
}
task upgradedClusterTest(type: RestIntegTestTask) {
dependsOn(mixedClusterTestRunner, 'oldClusterTestCluster#node0.stop')
}
Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask) {
dependsOn(mixedClusterTestRunner, "${baseName}#oldClusterTestCluster#node0.stop")
}
upgradedClusterTestCluster {
configure(extensions.findByName("${baseName}#upgradedClusterTestCluster")) {
distribution = 'zip'
clusterName = 'rolling-upgrade'
unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() }
dataDir = "${-> oldClusterTest.nodes[0].dataDir}"
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
}
}
upgradedClusterTestRunner {
Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner")
upgradedClusterTestRunner.configure {
systemProperty 'tests.rest.suite', 'upgraded_cluster'
// only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion
finalizedBy 'mixedClusterTestCluster#stop'
}
finalizedBy "${baseName}#mixedClusterTestCluster#stop"
}
task integTest {
Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") {
dependsOn = [upgradedClusterTest]
}
bwcTest.dependsOn(versionBwcTest)
}
test.enabled = false // no unit tests for rolling upgrades, only the rest integration test
check.dependsOn(integTest)
repositories {
maven {
url "https://oss.sonatype.org/content/repositories/snapshots/"
}
// basic integ tests includes testing bwc against the most recent version
task integTest {
dependsOn = ["v${wireCompatVersions[-1]}#bwcTest"]
}
check.dependsOn(integTest)

1
qa/smoke-test-ingest-with-all-dependencies/build.gradle
View File

@ -30,6 +30,5 @@ dependencies {
integTestCluster {
plugin ':plugins:ingest-geoip'
setting 'script.inline', 'true'
setting 'script.max_compilations_per_minute', '1000'
}

13
qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java
View File

@ -19,30 +19,25 @@
package org.elasticsearch.ingest;
import java.util.Arrays;
import java.util.Collections;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.script.ScriptContextRegistry;
import org.elasticsearch.script.ScriptEngineRegistry;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptSettings;
import org.elasticsearch.script.mustache.MustacheScriptEngine;
import org.elasticsearch.test.ESTestCase;
import org.junit.Before;
import java.util.Collections;
public abstract class AbstractScriptTestCase extends ESTestCase {
protected TemplateService templateService;
@Before
public void init() throws Exception {
ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(new MustacheScriptEngine()));
ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new MustacheScriptEngine()));
ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptEngineRegistry, scriptContextRegistry);
templateService = new InternalTemplateService(scriptService);
}

12
qa/vagrant/versions
View File

@ -1,12 +0,0 @@
5.0.0
5.0.1
5.0.2
5.1.1
5.1.2
5.2.0
5.2.1
5.2.2
5.3.0
5.3.1
5.3.2
5.4.0

2
settings.gradle
View File

@ -61,8 +61,8 @@ List projects = [
'plugins:jvm-example',
'plugins:store-smb',
'qa:auto-create-index',
'qa:backwards-5.0',
'qa:evil-tests',
'qa:mixed-cluster',
'qa:multi-cluster-search',
'qa:no-bootstrap-tests',
'qa:reindex-from-old',

8
test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java
View File

@ -42,14 +42,14 @@ public final class RandomDocumentPicks {
*/
public static String randomFieldName(Random random) {
int numLevels = RandomNumbers.randomIntBetween(random, 1, 5);
String fieldName = "";
StringBuilder fieldName = new StringBuilder();
for (int i = 0; i < numLevels; i++) {
if (i > 0) {
fieldName += ".";
fieldName.append('.');
}
fieldName += randomString(random);
fieldName.append(randomString(random));
}
return fieldName;
return fieldName.toString();
}
/**

7
test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
View File

@ -1013,10 +1013,9 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
new Class[]{Client.class},
clientInvocationHandler);
ScriptModule scriptModule = createScriptModule(pluginsService.filterPlugins(ScriptPlugin.class));
List<Setting<?>> scriptSettings = scriptModule.getSettings();
scriptSettings.addAll(pluginsService.getPluginSettings());
scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
SettingsModule settingsModule = new SettingsModule(nodeSettings, scriptSettings, pluginsService.getPluginSettingsFilter());
List<Setting<?>> additionalSettings = pluginsService.getPluginSettings();
additionalSettings.add(InternalSettingsPlugin.VERSION_CREATED);
SettingsModule settingsModule = new SettingsModule(nodeSettings, additionalSettings, pluginsService.getPluginSettingsFilter());
searchModule = new SearchModule(nodeSettings, false, pluginsService.filterPlugins(SearchPlugin.class));
IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class));
List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();

2
test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
View File

@ -1722,8 +1722,6 @@ public abstract class ESIntegTestCase extends ESTestCase {
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b")
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1b")
.put(ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE.getKey(), 1000)
.put("script.stored", "true")
.put("script.inline", "true")
// by default we never cache below 10k docs in a segment,
// bypass this limit so that caching gets some testing in
// integration tests that usually create few documents

10
test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java
View File

@ -107,6 +107,16 @@ public class VersionUtils {
return version;
}
/** Returns the {@link Version} before the {@link Version#CURRENT} where the minor version is less than the currents minor version. */
public static Version getPreviousMinorVersion() {
Version version = Version.CURRENT;
do {
version = getPreviousVersion(version);
assert version.before(Version.CURRENT);
} while (version.minor == Version.CURRENT.minor);
return version;
}
/** Returns the oldest {@link Version} */
public static Version getFirstVersion() {
return RELEASED_VERSIONS.get(0);

20
test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
View File

@ -196,21 +196,25 @@ public class ElasticsearchAssertions {
}
public static String formatShardStatus(BroadcastResponse response) {
String msg = " Total shards: " + response.getTotalShards() + " Successful shards: " + response.getSuccessfulShards() + " & "
+ response.getFailedShards() + " shard failures:";
StringBuilder msg = new StringBuilder();
msg.append(" Total shards: ").append(response.getTotalShards())
.append(" Successful shards: ").append(response.getSuccessfulShards())
.append(" & ").append(response.getFailedShards()).append(" shard failures:");
for (ShardOperationFailedException failure : response.getShardFailures()) {
msg += "\n " + failure.toString();
msg.append("\n ").append(failure);
}
return msg;
return msg.toString();
}
public static String formatShardStatus(SearchResponse response) {
String msg = " Total shards: " + response.getTotalShards() + " Successful shards: " + response.getSuccessfulShards() + " & "
+ response.getFailedShards() + " shard failures:";
StringBuilder msg = new StringBuilder();
msg.append(" Total shards: ").append(response.getTotalShards())
.append(" Successful shards: ").append(response.getSuccessfulShards())
.append(" & ").append(response.getFailedShards()).append(" shard failures:");
for (ShardSearchFailure failure : response.getShardFailures()) {
msg += "\n " + failure.toString();
msg.append("\n ").append(failure);
}
return msg;
return msg.toString();
}
public static void assertNoSearchHits(SearchResponse searchResponse) {