honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into wildcard_imports

This commit is contained in:
Ryan Ernst 2015-12-18 12:24:30 -08:00
parent 0f518e1b07 10dfa32f9d
commit 853e9c0fd1
734 changed files with 17498 additions and 7646 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
.gitignore vendored
View File

@ -1,37 +1,42 @@
# intellij files
.idea/
.gradle/
*.iml
*.ipr
*.iws
work/
/data/
logs/
.DS_Store
build/
generated-resources/
**/.local*
docs/html/
docs/build.log
/tmp/
backwards/
html_docs
.vagrant/
## eclipse ignores (use 'mvn eclipse:eclipse' to build eclipse projects)
## All files (.project, .classpath, .settings/*) should be generated through Maven which
## will correctly set the classpath based on the declared dependencies and write settings
## files to ensure common coding style across Eclipse and IDEA.
# eclipse files
.project
.classpath
eclipse-build
.settings
## netbeans ignores
# netbeans files
nb-configuration.xml
nbactions.xml
dependency-reduced-pom.xml
# gradle stuff
.gradle/
build/
generated-resources/
# old patterns specific to maven
# maven stuff (to be removed when trunk becomes 4.x)
*-execution-hints.log
target/
dependency-reduced-pom.xml
# testing stuff
**/.local*
.vagrant/
# osx stuff
.DS_Store
# needed in case docs build is run...maybe we can configure doc build to generate files under build?
html_docs
# random old stuff that we should look at the necessity of...
/tmp/
backwards/

4
TESTING.asciidoc
View File

@ -290,14 +290,14 @@ The REST tests are run automatically when executing the "gradle check" command.
REST tests use the following command:
---------------------------------------------------------------------------
gradle :distribution:tar:integTest \
gradle :distribution:integ-test-zip:integTest \
-Dtests.class=org.elasticsearch.test.rest.RestIT
---------------------------------------------------------------------------
A specific test case can be run with
---------------------------------------------------------------------------
gradle :distribution:tar:integTest \
gradle :distribution:integ-test-zip:integTest \
-Dtests.class=org.elasticsearch.test.rest.RestIT \
-Dtests.method="test {p0=cat.shards/10_basic/Help}"
---------------------------------------------------------------------------

31
build.gradle
View File

@ -97,6 +97,7 @@ subprojects {
// the "value" -quiet is added, separated by a space. This is ok since the javadoc
// command already adds -quiet, so we are just duplicating it
// see https://discuss.gradle.org/t/add-custom-javadoc-option-that-does-not-take-an-argument/5959
javadoc.options.encoding='UTF8'
javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet')
}
}
@ -108,7 +109,7 @@ subprojects {
ext.projectSubstitutions = [
"org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
"org.elasticsearch:elasticsearch:${version}": ':core',
"org.elasticsearch:test-framework:${version}": ':test-framework',
"org.elasticsearch.test:framework:${version}": ':test:framework',
"org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:integ-test-zip',
"org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip',
"org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar',
@ -130,8 +131,8 @@ subprojects {
// the dependency is added.
gradle.projectsEvaluated {
allprojects {
if (project.path == ':test-framework') {
// :test-framework:test cannot run before and after :core:test
if (project.path == ':test:framework') {
// :test:framework:test cannot run before and after :core:test
return
}
configurations.all {
@ -168,6 +169,30 @@ gradle.projectsEvaluated {
// intellij configuration
allprojects {
apply plugin: 'idea'
idea {
module {
// same as for the IntelliJ Gradle tooling integration
inheritOutputDirs = false
outputDir = file('build/classes/main')
testOutputDir = file('build/classes/test')
iml {
// fix so that Gradle idea plugin properly generates support for resource folders
// see also https://issues.gradle.org/browse/GRADLE-2975
withXml {
it.asNode().component.content.sourceFolder.findAll { it.@url == 'file://$MODULE_DIR$/src/main/resources' }.each {
it.attributes().remove('isTestSource')
it.attributes().put('type', 'java-resource')
}
it.asNode().component.content.sourceFolder.findAll { it.@url == 'file://$MODULE_DIR$/src/test/resources' }.each {
it.attributes().remove('isTestSource')
it.attributes().put('type', 'java-test-resource')
}
}
}
}
}
}
idea {

1
buildSrc/build.gradle
View File

@ -63,6 +63,7 @@ dependencies {
compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
compile 'de.thetaphi:forbiddenapis:2.0'
compile 'com.bmuschko:gradle-nexus-plugin:2.3.1'
compile 'org.apache.rat:apache-rat:0.11'
}
processResources {

8
buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy
View File

@ -78,7 +78,7 @@ class RandomizedTestingTask extends DefaultTask {
@Input
String argLine = null
Map<String, String> systemProperties = new HashMap<>()
Map<String, Object> systemProperties = new HashMap<>()
PatternFilterable patternSet = new PatternSet()
RandomizedTestingTask() {
@ -100,7 +100,7 @@ class RandomizedTestingTask extends DefaultTask {
jvmArgs.add(argument)
}
void systemProperty(String property, String value) {
void systemProperty(String property, Object value) {
systemProperties.put(property, value)
}
@ -245,8 +245,8 @@ class RandomizedTestingTask extends DefaultTask {
exclude(name: excludePattern)
}
}
for (Map.Entry<String, String> prop : systemProperties) {
sysproperty key: prop.getKey(), value: prop.getValue()
for (Map.Entry<String, Object> prop : systemProperties) {
sysproperty key: prop.getKey(), value: prop.getValue().toString()
}
makeListeners()
}

105
buildSrc/src/main/groovy/org/elasticsearch/gradle/AntTask.groovy Normal file
View File

@ -0,0 +1,105 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle
import org.apache.tools.ant.BuildException
import org.apache.tools.ant.BuildListener
import org.apache.tools.ant.BuildLogger
import org.apache.tools.ant.DefaultLogger
import org.apache.tools.ant.Project
import org.gradle.api.DefaultTask
import org.gradle.api.GradleException
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.Optional
import org.gradle.api.tasks.TaskAction
import java.nio.charset.Charset
/**
* A task which will run ant commands.
*
* Logging for the task is customizable for subclasses by overriding makeLogger.
*/
public class AntTask extends DefaultTask {
/**
* A buffer that will contain the output of the ant code run,
* if the output was not already written directly to stdout.
*/
public final ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream()
@TaskAction
final void executeTask() {
AntBuilder ant = new AntBuilder()
// remove existing loggers, we add our own
List<BuildLogger> toRemove = new ArrayList<>();
for (BuildListener listener : ant.project.getBuildListeners()) {
if (l instanceof BuildLogger) {
toRemove.add(listener);
}
}
for (BuildLogger listener : toRemove) {
ant.project.removeBuildListener(listener)
}
final int outputLevel = logger.isDebugEnabled() ? Project.MSG_DEBUG : (logger.isInfoEnabled() ? Project.MSG_INFO : Project.MSG_WARN)
final PrintStream stream = useStdout() ? System.out : new PrintStream(outputBuffer, true, Charset.defaultCharset().name())
BuildLogger antLogger = makeLogger(stream, outputLevel)
ant.project.addBuildListener(antLogger)
try {
runAnt(ant)
} catch (BuildException e) {
// ant failed, so see if we have buffered output to emit, then rethrow the failure
String buffer = outputBuffer.toString()
if (buffer.isEmpty() == false) {
logger.error("=== Ant output ===\n${buffer}")
}
throw e
}
}
/** Runs the doAnt closure. This can be overridden by subclasses instead of having to set a closure. */
protected void runAnt(AntBuilder ant) {
if (doAnt == null) {
throw new GradleException("Missing doAnt for ${name}")
}
doAnt(ant)
}
/** Create the logger the ant runner will use, with the given stream for error/output. */
protected BuildLogger makeLogger(PrintStream stream, int outputLevel) {
return new DefaultLogger(
errorPrintStream: stream,
outputPrintStream: stream,
messageOutputLevel: outputLevel)
}
/**
* Returns true if the ant logger should write to stdout, or false if to the buffer.
* The default implementation writes to the buffer when gradle info logging is disabled.
*/
protected boolean useStdout() {
return logger.isInfoEnabled()
}
}

3
buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View File

@ -202,7 +202,7 @@ class BuildPlugin implements Plugin<Project> {
// force all dependencies added directly to compile/testCompile to be non-transitive, except for ES itself
Closure disableTransitiveDeps = { ModuleDependency dep ->
if (!(dep instanceof ProjectDependency) && dep.getGroup() != 'org.elasticsearch') {
if (!(dep instanceof ProjectDependency) && dep.group.startsWith('org.elasticsearch') == false) {
dep.transitive = false
// also create a configuration just for this dependency version, so that later
@ -302,6 +302,7 @@ class BuildPlugin implements Plugin<Project> {
options.compilerArgs << '-profile' << project.compactProfile
}
options.encoding = 'UTF-8'
//options.incremental = true
}
}
}

2
buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
View File

@ -60,7 +60,7 @@ public class PluginBuildPlugin extends BuildPlugin {
private static void configureDependencies(Project project) {
project.dependencies {
provided "org.elasticsearch:elasticsearch:${project.versions.elasticsearch}"
testCompile "org.elasticsearch:test-framework:${project.versions.elasticsearch}"
testCompile "org.elasticsearch.test:framework:${project.versions.elasticsearch}"
// we "upgrade" these optional deps to provided for plugins, since they will run
// with a full elasticsearch server that includes optional deps
provided "com.spatial4j:spatial4j:${project.versions.spatial4j}"

5
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ForbiddenPatternsTask.groovy
View File

@ -62,11 +62,14 @@ public class ForbiddenPatternsTask extends DefaultTask {
patterns.put('nocommit', /nocommit/)
patterns.put('tab', /\t/)
patterns.put('wildcard imports', /^\s*import.*\.\*/)
inputs.property("excludes", filesFilter.excludes)
inputs.property("rules", patterns)
}
/** Adds a file glob pattern to be excluded */
public void exclude(String... excludes) {
this.filesFilter.exclude(excludes)
filesFilter.exclude(excludes)
}
/** Adds a pattern to forbid. T */

122
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy Normal file
View File

@ -0,0 +1,122 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.precommit
import org.apache.rat.anttasks.Report
import org.apache.rat.anttasks.SubstringLicenseMatcher
import org.apache.rat.license.SimpleLicenseFamily
import org.elasticsearch.gradle.AntTask
import org.gradle.api.tasks.SourceSet
import java.nio.file.Files
/**
* Checks files for license headers.
* <p>
* This is a port of the apache lucene check
*/
public class LicenseHeadersTask extends AntTask {
LicenseHeadersTask() {
description = "Checks sources for missing, incorrect, or unacceptable license headers"
}
@Override
protected void runAnt(AntBuilder ant) {
ant.project.addTaskDefinition('ratReport', Report)
ant.project.addDataTypeDefinition('substringMatcher', SubstringLicenseMatcher)
ant.project.addDataTypeDefinition('approvedLicense', SimpleLicenseFamily)
// create a file for the log to go to under reports/
File reportDir = new File(project.buildDir, "reports/licenseHeaders")
reportDir.mkdirs()
File reportFile = new File(reportDir, "rat.log")
Files.deleteIfExists(reportFile.toPath())
// run rat, going to the file
ant.ratReport(reportFile: reportFile.absolutePath, addDefaultLicenseMatchers: true) {
// checks all the java sources (allJava)
for (SourceSet set : project.sourceSets) {
for (File dir : set.allJava.srcDirs) {
// sometimes these dirs don't exist, e.g. site-plugin has no actual java src/main...
if (dir.exists()) {
ant.fileset(dir: dir)
}
}
}
// BSD 4-clause stuff (is disallowed below)
// we keep this here, in case someone adds BSD code for some reason, it should never be allowed.
substringMatcher(licenseFamilyCategory: "BSD4 ",
licenseFamilyName: "Original BSD License (with advertising clause)") {
pattern(substring: "All advertising materials")
}
// Apache
substringMatcher(licenseFamilyCategory: "AL ",
licenseFamilyName: "Apache") {
// Apache license (ES)
pattern(substring: "Licensed to Elasticsearch under one or more contributor")
// Apache license (ASF)
pattern(substring: "Licensed to the Apache Software Foundation (ASF) under")
// this is the old-school one under some files
pattern(substring: "Licensed under the Apache License, Version 2.0 (the \"License\")")
}
// Generated resources
substringMatcher(licenseFamilyCategory: "GEN ",
licenseFamilyName: "Generated") {
// parsers generated by antlr
pattern(substring: "ANTLR GENERATED CODE")
}
// approved categories
approvedLicense(familyName: "Apache")
approvedLicense(familyName: "Generated")
}
// check the license file for any errors, this should be fast.
boolean zeroUnknownLicenses = false
boolean foundProblemsWithFiles = false
reportFile.eachLine('UTF-8') { line ->
if (line.startsWith("0 Unknown Licenses")) {
zeroUnknownLicenses = true
}
if (line.startsWith(" !")) {
foundProblemsWithFiles = true
}
}
if (zeroUnknownLicenses == false || foundProblemsWithFiles) {
// print the unapproved license section, usually its all you need to fix problems.
int sectionNumber = 0
reportFile.eachLine('UTF-8') { line ->
if (line.startsWith("*******************************")) {
sectionNumber++
} else {
if (sectionNumber == 2) {
logger.error(line)
}
}
}
throw new IllegalStateException("License header problems were found! Full details: " + reportFile.absolutePath)
}
}
}

4
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
View File

@ -34,7 +34,9 @@ class PrecommitTasks {
List<Task> precommitTasks = [
configureForbiddenApis(project),
project.tasks.create('forbiddenPatterns', ForbiddenPatternsTask.class),
project.tasks.create('jarHell', JarHellTask.class)]
project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
project.tasks.create('jarHell', JarHellTask.class),
project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)]
// tasks with just tests don't need dependency licenses, so this flag makes adding
// the task optional

205
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy Normal file
View File

@ -0,0 +1,205 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.precommit
import org.apache.tools.ant.BuildLogger
import org.apache.tools.ant.DefaultLogger
import org.apache.tools.ant.Project
import org.elasticsearch.gradle.AntTask
import org.gradle.api.artifacts.Configuration
import org.gradle.api.file.FileCollection
import java.nio.file.FileVisitResult
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.SimpleFileVisitor
import java.nio.file.attribute.BasicFileAttributes
/**
* Basic static checking to keep tabs on third party JARs
*/
public class ThirdPartyAuditTask extends AntTask {
// true to be lenient about MISSING CLASSES
private boolean missingClasses;
// patterns for classes to exclude, because we understand their issues
private String[] excludes = new String[0];
ThirdPartyAuditTask() {
dependsOn(project.configurations.testCompile)
description = "Checks third party JAR bytecode for missing classes, use of internal APIs, and other horrors'"
}
/**
* Set to true to be lenient with missing classes. By default this check will fail if it finds
* MISSING CLASSES. This means the set of jars is incomplete. However, in some cases
* this can be due to intentional exclusions that are well-tested and understood.
*/
public void setMissingClasses(boolean value) {
missingClasses = value;
}
/**
* Returns true if leniency about missing classes is enabled.
*/
public boolean isMissingClasses() {
return missingClasses;
}
/**
* classes that should be excluded from the scan,
* e.g. because we know what sheisty stuff those particular classes are up to.
*/
public void setExcludes(String[] classes) {
for (String s : classes) {
if (s.indexOf('*') != -1) {
throw new IllegalArgumentException("illegal third party audit exclusion: '" + s + "', wildcards are not permitted!")
}
}
excludes = classes;
}
/**
* Returns current list of exclusions.
*/
public String[] getExcludes() {
return excludes;
}
@Override
protected BuildLogger makeLogger(PrintStream stream, int outputLevel) {
return new DefaultLogger(
errorPrintStream: stream,
outputPrintStream: stream,
// ignore passed in outputLevel for now, until we are filtering warning messages
messageOutputLevel: Project.MSG_ERR)
}
@Override
protected void runAnt(AntBuilder ant) {
ant.project.addTaskDefinition('thirdPartyAudit', de.thetaphi.forbiddenapis.ant.AntTask)
// we only want third party dependencies.
FileCollection jars = project.configurations.testCompile.fileCollection({ dependency ->
dependency.group.startsWith("org.elasticsearch") == false
})
// we don't want provided dependencies, which we have already scanned. e.g. don't
// scan ES core's dependencies for every single plugin
Configuration provided = project.configurations.findByName('provided')
if (provided != null) {
jars -= provided
}
// no dependencies matched, we are done
if (jars.isEmpty()) {
return;
}
// print which jars we are going to scan, always
// this is not the time to try to be succinct! Forbidden will print plenty on its own!
Set<String> names = new HashSet<>()
for (File jar : jars) {
names.add(jar.getName())
}
logger.error("[thirdPartyAudit] Scanning: " + names)
// warn that classes are missing
// TODO: move these to excludes list!
if (missingClasses) {
logger.warn("[thirdPartyAudit] WARNING: CLASSES ARE MISSING! Expect NoClassDefFoundError in bug reports from users!")
}
// TODO: forbidden-apis + zipfileset gives O(n^2) behavior unless we dump to a tmpdir first,
// and then remove our temp dir afterwards. don't complain: try it yourself.
// we don't use gradle temp dir handling, just google it, or try it yourself.
File tmpDir = new File(project.buildDir, 'tmp/thirdPartyAudit')
// clean up any previous mess (if we failed), then unzip everything to one directory
ant.delete(dir: tmpDir.getAbsolutePath())
tmpDir.mkdirs()
for (File jar : jars) {
ant.unzip(src: jar.getAbsolutePath(), dest: tmpDir.getAbsolutePath())
}
// convert exclusion class names to binary file names
String[] excludedFiles = new String[excludes.length];
for (int i = 0; i < excludes.length; i++) {
excludedFiles[i] = excludes[i].replace('.', '/') + ".class"
// check if the excluded file exists, if not, sure sign things are outdated
if (! new File(tmpDir, excludedFiles[i]).exists()) {
throw new IllegalStateException("bogus thirdPartyAudit exclusion: '" + excludes[i] + "', not found in any dependency")
}
}
// jarHellReprise
checkSheistyClasses(tmpDir.toPath(), new HashSet<>(Arrays.asList(excludedFiles)));
ant.thirdPartyAudit(internalRuntimeForbidden: true,
failOnUnsupportedJava: false,
failOnMissingClasses: !missingClasses,
classpath: project.configurations.testCompile.asPath) {
fileset(dir: tmpDir, excludes: excludedFiles.join(','))
}
// clean up our mess (if we succeed)
ant.delete(dir: tmpDir.getAbsolutePath())
}
/**
* check for sheisty classes: if they also exist in the extensions classloader, its jar hell with the jdk!
*/
private void checkSheistyClasses(Path root, Set<String> excluded) {
// system.parent = extensions loader.
// note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem!).
// but groovy/gradle needs to work at all first!
ClassLoader ext = ClassLoader.getSystemClassLoader().getParent()
assert ext != null
Set<String> sheistySet = new TreeSet<>();
Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
String entry = root.relativize(file).toString()
if (entry.endsWith(".class")) {
if (ext.getResource(entry) != null) {
sheistySet.add(entry);
}
}
return FileVisitResult.CONTINUE;
}
});
// check if we are ok
if (sheistySet.isEmpty()) {
return;
}
// leniency against exclusions list
sheistySet.removeAll(excluded);
if (sheistySet.isEmpty()) {
logger.warn("[thirdPartyAudit] WARNING: JAR HELL WITH JDK! Expect insanely hard-to-debug problems!")
} else {
throw new IllegalStateException("JAR HELL WITH JDK! " + sheistySet);
}
}
}

6
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
View File

@ -33,10 +33,10 @@ class ClusterConfiguration {
int numNodes = 1
@Input
int baseHttpPort = 9400
int httpPort = 0
@Input
int baseTransportPort = 9500
int transportPort = 0
@Input
boolean daemonize = true
@ -55,7 +55,7 @@ class ClusterConfiguration {
@Input
Closure waitCondition = { NodeInfo node, AntBuilder ant ->
File tmpFile = new File(node.cwd, 'wait.success')
ant.get(src: "http://localhost:${node.httpPort()}",
ant.get(src: "http://${node.httpUri()}",
dest: tmpFile.toString(),
ignoreerrors: true, // do not fail on error, so logging buffers can be flushed by the wait task
retries: 10)

43
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
View File

@ -38,8 +38,10 @@ class ClusterFormationTasks {
/**
* Adds dependent tasks to the given task to start and stop a cluster with the given configuration.
*
* Returns an object that will resolve at execution time of the given task to a uri for the cluster.
*/
static void setup(Project project, Task task, ClusterConfiguration config) {
static Object setup(Project project, Task task, ClusterConfiguration config) {
if (task.getEnabled() == false) {
// no need to add cluster formation tasks if the task won't run!
return
@ -55,6 +57,9 @@ class ClusterFormationTasks {
Task wait = configureWaitTask("${task.name}#wait", project, nodes, startTasks)
task.dependsOn(wait)
// delay the resolution of the uri by wrapping in a closure, so it is not used until read for tests
return "${-> nodes[0].transportUri()}"
}
/** Adds a dependency on the given distribution */
@ -200,17 +205,24 @@ class ClusterFormationTasks {
/** Adds a task to write elasticsearch.yml for the given node configuration */
static Task configureWriteConfigTask(String name, Project project, Task setup, NodeInfo node) {
Map esConfig = [
'cluster.name' : node.clusterName,
'http.port' : node.httpPort(),
'transport.tcp.port' : node.transportPort(),
'pidfile' : node.pidFile,
'discovery.zen.ping.unicast.hosts': (0..<node.config.numNodes).collect{"127.0.0.1:${node.config.baseTransportPort + it}"}.join(','),
'path.repo' : "${node.homeDir}/repo",
'path.shared_data' : "${node.homeDir}/../",
// Define a node attribute so we can test that it exists
'node.testattr' : 'test',
'repositories.url.allowed_urls' : 'http://snapshot.test*'
'cluster.name' : node.clusterName,
'pidfile' : node.pidFile,
'path.repo' : "${node.homeDir}/repo",
'path.shared_data' : "${node.homeDir}/../",
// Define a node attribute so we can test that it exists
'node.testattr' : 'test',
'repositories.url.allowed_urls': 'http://snapshot.test*'
]
if (node.config.numNodes == 1) {
esConfig['http.port'] = node.config.httpPort
esConfig['transport.tcp.port'] = node.config.transportPort
} else {
// TODO: fix multi node so it doesn't use hardcoded prots
esConfig['http.port'] = 9400 + node.nodeNum
esConfig['transport.tcp.port'] = 9500 + node.nodeNum
esConfig['discovery.zen.ping.unicast.hosts'] = (0..<node.config.numNodes).collect{"localhost:${9500 + it}"}.join(',')
}
esConfig.putAll(node.config.settings)
Task writeConfig = project.tasks.create(name: name, type: DefaultTask, dependsOn: setup)
@ -400,7 +412,12 @@ class ClusterFormationTasks {
resourceexists {
file(file: node.pidFile.toString())
}
socket(server: '127.0.0.1', port: node.httpPort())
resourceexists {
file(file: node.httpPortsFile.toString())
}
resourceexists {
file(file: node.transportPortsFile.toString())
}
}
}
}
@ -444,6 +461,8 @@ class ClusterFormationTasks {
logger.error("|-----------------------------------------")
logger.error("| failure marker exists: ${node.failedMarker.exists()}")
logger.error("| pid file exists: ${node.pidFile.exists()}")
logger.error("| http ports file exists: ${node.httpPortsFile.exists()}")
logger.error("| transport ports file exists: ${node.transportPortsFile.exists()}")
// the waitfor failed, so dump any output we got (if info logging this goes directly to stdout)
logger.error("|\n| [ant output]")
node.buffer.toString('UTF-8').eachLine { line -> logger.error("| ${line}") }

23
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
View File

@ -43,6 +43,12 @@ class NodeInfo {
/** the pid file the node will use */
File pidFile
/** a file written by elasticsearch containing the ports of each bound address for http */
File httpPortsFile
/** a file written by elasticsearch containing the ports of each bound address for transport */
File transportPortsFile
/** elasticsearch home dir */
File homeDir
@ -92,6 +98,10 @@ class NodeInfo {
homeDir = homeDir(baseDir, config.distribution)
confDir = confDir(baseDir, config.distribution)
configFile = new File(confDir, 'elasticsearch.yml')
// even for rpm/deb, the logs are under home because we dont start with real services
File logsDir = new File(homeDir, 'logs')
httpPortsFile = new File(logsDir, 'http.ports')
transportPortsFile = new File(logsDir, 'transport.ports')
cwd = new File(baseDir, "cwd")
failedMarker = new File(cwd, 'run.failed')
startLog = new File(cwd, 'run.log')
@ -119,6 +129,7 @@ class NodeInfo {
'JAVA_HOME' : project.javaHome,
'ES_GC_OPTS': config.jvmArgs // we pass these with the undocumented gc opts so the argline can set gc, etc
]
args.add("-Des.tests.portsfile=true")
args.addAll(config.systemProperties.collect { key, value -> "-D${key}=${value}" })
for (Map.Entry<String, String> property : System.properties.entrySet()) {
if (property.getKey().startsWith('es.')) {
@ -159,14 +170,14 @@ class NodeInfo {
wrapperScript.setText("\"${esScript}\" ${argsPasser} > run.log 2>&1 ${exitMarker}", 'UTF-8')
}
/** Returns the http port for this node */
int httpPort() {
return config.baseHttpPort + nodeNum
/** Returns an address and port suitable for a uri to connect to this node over http */
String httpUri() {
return httpPortsFile.readLines("UTF-8").get(0)
}
/** Returns the transport port for this node */
int transportPort() {
return config.baseTransportPort + nodeNum
/** Returns an address and port suitable for a uri to connect to this node over transport protocol */
String transportUri() {
return transportPortsFile.readLines("UTF-8").get(0)
}
/** Returns the directory elasticsearch home is contained in for the given distribution */

4
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
View File

@ -57,12 +57,12 @@ public class RestIntegTestTask extends RandomizedTestingTask {
RestSpecHack.configureDependencies(project)
project.afterEvaluate {
dependsOn(RestSpecHack.configureTask(project, includePackaged))
systemProperty('tests.cluster', "localhost:${clusterConfig.baseTransportPort}")
}
// this must run after all projects have been configured, so we know any project
// references can be accessed as a fully configured
project.gradle.projectsEvaluated {
ClusterFormationTasks.setup(project, this, clusterConfig)
Object clusterUri = ClusterFormationTasks.setup(project, this, clusterConfig)
systemProperty('tests.cluster', clusterUri)
}
}

2
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy
View File

@ -8,7 +8,7 @@ import org.gradle.util.ConfigureUtil
public class RunTask extends DefaultTask {
ClusterConfiguration clusterConfig = new ClusterConfiguration(baseHttpPort: 9200, baseTransportPort: 9300, daemonize: false)
ClusterConfiguration clusterConfig = new ClusterConfiguration(httpPort: 9200, transportPort: 9300, daemonize: false)
public RunTask() {
description = "Runs elasticsearch with '${project.path}'"

2
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy
View File

@ -42,7 +42,7 @@ public class StandaloneTestBasePlugin implements Plugin<Project> {
// only setup tests to build
project.sourceSets.create('test')
project.dependencies.add('testCompile', "org.elasticsearch:test-framework:${VersionProperties.elasticsearch}")
project.dependencies.add('testCompile', "org.elasticsearch.test:framework:${VersionProperties.elasticsearch}")
project.eclipse.classpath.sourceSets = [project.sourceSets.test]
project.eclipse.classpath.plusConfigurations = [project.configurations.testRuntime]

3
buildSrc/src/main/resources/forbidden/all-signatures.txt
View File

@ -123,3 +123,6 @@ java.lang.System#getProperties() @ Use BootstrapInfo.getSystemProperties for a r
java.util.Collections#EMPTY_LIST
java.util.Collections#EMPTY_MAP
java.util.Collections#EMPTY_SET
java.util.Collections#shuffle(java.util.List) @ Use java.util.Collections#shuffle(java.util.List, java.util.Random) with a reproducible source of randomness
java.util.Random#<init>() @ Use org.elasticsearch.common.random.Randomness#create for reproducible sources of randomness

10
core/build.gradle
View File

@ -82,7 +82,7 @@ dependencies {
compile "net.java.dev.jna:jna:${versions.jna}", optional
if (isEclipse == false || project.path == ":core-tests") {
testCompile("org.elasticsearch:test-framework:${version}") {
testCompile("org.elasticsearch.test:framework:${version}") {
// tests use the locally compiled version of core
exclude group: 'org.elasticsearch', module: 'elasticsearch'
}
@ -111,6 +111,14 @@ forbiddenPatterns {
exclude '**/org/elasticsearch/cluster/routing/shard_routes.txt'
}
// classes are missing, e.g. org.jboss.marshalling.Marshaller
thirdPartyAudit.missingClasses = true
// uses internal sun ssl classes!
thirdPartyAudit.excludes = [
// uses internal java api: sun.security.x509 (X509CertInfo, X509CertImpl, X500Name)
'org.jboss.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',
]
// dependency license are currently checked in distribution
dependencyLicenses.enabled = false

1
core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java
View File

@ -54,7 +54,6 @@ public class MapperQueryParser extends QueryParser {
static {
Map<String, FieldQueryExtension> fieldQueryExtensions = new HashMap<>();
fieldQueryExtensions.put(ExistsFieldQueryExtension.NAME, new ExistsFieldQueryExtension());
fieldQueryExtensions.put(MissingFieldQueryExtension.NAME, new MissingFieldQueryExtension());
FIELD_QUERY_EXTENSIONS = unmodifiableMap(fieldQueryExtensions);
}

12
core/src/main/java/org/elasticsearch/Version.java
View File

@ -268,11 +268,15 @@ public class Version {
public static final int V_2_0_1_ID = 2000199;
public static final Version V_2_0_1 = new Version(V_2_0_1_ID, false, org.apache.lucene.util.Version.LUCENE_5_2_1);
public static final int V_2_0_2_ID = 2000299;
public static final Version V_2_0_2 = new Version(V_2_0_2_ID, true, org.apache.lucene.util.Version.LUCENE_5_2_1);
public static final Version V_2_0_2 = new Version(V_2_0_2_ID, false, org.apache.lucene.util.Version.LUCENE_5_2_1);
public static final int V_2_0_3_ID = 2000399;
public static final Version V_2_0_3 = new Version(V_2_0_3_ID, true, org.apache.lucene.util.Version.LUCENE_5_2_1);
public static final int V_2_1_0_ID = 2010099;
public static final Version V_2_1_0 = new Version(V_2_1_0_ID, false, org.apache.lucene.util.Version.LUCENE_5_3_1);
public static final int V_2_1_1_ID = 2010199;
public static final Version V_2_1_1 = new Version(V_2_1_1_ID, true, org.apache.lucene.util.Version.LUCENE_5_3_1);
public static final Version V_2_1_1 = new Version(V_2_1_1_ID, false, org.apache.lucene.util.Version.LUCENE_5_3_1);
public static final int V_2_1_2_ID = 2010299;
public static final Version V_2_1_2 = new Version(V_2_1_2_ID, true, org.apache.lucene.util.Version.LUCENE_5_3_1);
public static final int V_2_2_0_ID = 2020099;
public static final Version V_2_2_0 = new Version(V_2_2_0_ID, true, org.apache.lucene.util.Version.LUCENE_5_4_0);
public static final int V_3_0_0_ID = 3000099;
@ -293,10 +297,14 @@ public class Version {
return V_3_0_0;
case V_2_2_0_ID:
return V_2_2_0;
case V_2_1_2_ID:
return V_2_1_2;
case V_2_1_1_ID:
return V_2_1_1;
case V_2_1_0_ID:
return V_2_1_0;
case V_2_0_3_ID:
return V_2_0_3;
case V_2_0_2_ID:
return V_2_0_2;
case V_2_0_1_ID:

3
core/src/main/java/org/elasticsearch/action/ActionModule.java
View File

@ -107,6 +107,8 @@ import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresAction;
import org.elasticsearch.action.admin.indices.shards.TransportIndicesShardStoresAction;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
import org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction;
import org.elasticsearch.action.admin.indices.flush.SyncedFlushAction;
import org.elasticsearch.action.admin.indices.flush.TransportSyncedFlushAction;
import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateAction;
import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteIndexTemplateAction;
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesAction;
@ -293,6 +295,7 @@ public class ActionModule extends AbstractModule {
registerAction(ValidateQueryAction.INSTANCE, TransportValidateQueryAction.class);
registerAction(RefreshAction.INSTANCE, TransportRefreshAction.class);
registerAction(FlushAction.INSTANCE, TransportFlushAction.class);
registerAction(SyncedFlushAction.INSTANCE, TransportSyncedFlushAction.class);
registerAction(ForceMergeAction.INSTANCE, TransportForceMergeAction.class);
registerAction(UpgradeAction.INSTANCE, TransportUpgradeAction.class);
registerAction(UpgradeStatusAction.INSTANCE, TransportUpgradeStatusAction.class);

127
core/src/main/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdater.java Normal file
View File

@ -0,0 +1,127 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.admin.cluster.settings;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlocks;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import static org.elasticsearch.cluster.ClusterState.builder;
/**
* Updates transient and persistent cluster state settings if there are any changes
* due to the update.
*/
final class SettingsUpdater {
final Settings.Builder transientUpdates = Settings.settingsBuilder();
final Settings.Builder persistentUpdates = Settings.settingsBuilder();
private final ClusterSettings clusterSettings;
SettingsUpdater(ClusterSettings clusterSettings) {
this.clusterSettings = clusterSettings;
}
synchronized Settings getTransientUpdates() {
return transientUpdates.build();
}
synchronized Settings getPersistentUpdate() {
return persistentUpdates.build();
}
synchronized ClusterState updateSettings(final ClusterState currentState, Settings transientToApply, Settings persistentToApply) {
boolean changed = false;
Settings.Builder transientSettings = Settings.settingsBuilder();
transientSettings.put(currentState.metaData().transientSettings());
changed |= apply(transientToApply, transientSettings, transientUpdates, "transient");
Settings.Builder persistentSettings = Settings.settingsBuilder();
persistentSettings.put(currentState.metaData().persistentSettings());
changed |= apply(persistentToApply, persistentSettings, persistentUpdates, "persistent");
if (!changed) {
return currentState;
}
MetaData.Builder metaData = MetaData.builder(currentState.metaData())
.persistentSettings(persistentSettings.build())
.transientSettings(transientSettings.build());
ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
boolean updatedReadOnly = MetaData.SETTING_READ_ONLY_SETTING.get(metaData.persistentSettings()) || MetaData.SETTING_READ_ONLY_SETTING.get(metaData.transientSettings());
if (updatedReadOnly) {
blocks.addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK);
} else {
blocks.removeGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK);
}
ClusterState build = builder(currentState).metaData(metaData).blocks(blocks).build();
Settings settings = build.metaData().settings();
// now we try to apply things and if they are invalid we fail
// this dryRun will validate & parse settings but won't actually apply them.
clusterSettings.dryRun(settings);
return build;
}
private boolean apply(Settings toApply, Settings.Builder target, Settings.Builder updates, String type) {
boolean changed = false;
final Set<String> toRemove = new HashSet<>();
Settings.Builder settingsBuilder = Settings.settingsBuilder();
for (Map.Entry<String, String> entry : toApply.getAsMap().entrySet()) {
if (entry.getValue() == null) {
toRemove.add(entry.getKey());
} else if (clusterSettings.isLoggerSetting(entry.getKey()) || clusterSettings.hasDynamicSetting(entry.getKey())) {
settingsBuilder.put(entry.getKey(), entry.getValue());
updates.put(entry.getKey(), entry.getValue());
changed = true;
} else {
throw new IllegalArgumentException(type + " setting [" + entry.getKey() + "], not dynamically updateable");
}
}
changed |= applyDeletes(toRemove, target);
target.put(settingsBuilder.build());
return changed;
}
private final boolean applyDeletes(Set<String> deletes, Settings.Builder builder) {
boolean changed = false;
for (String entry : deletes) {
Set<String> keysToRemove = new HashSet<>();
Set<String> keySet = builder.internalMap().keySet();
for (String key : keySet) {
if (Regex.simpleMatch(entry, key)) {
keysToRemove.add(key);
}
}
for (String key : keysToRemove) {
builder.remove(key);
changed = true;
}
}
return changed;
}
}

84
core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
View File

@ -28,23 +28,19 @@ import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.block.ClusterBlocks;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.allocation.AllocationService;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
import org.elasticsearch.cluster.settings.ClusterDynamicSettings;
import org.elasticsearch.cluster.settings.DynamicSettings;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.util.Map;
import static org.elasticsearch.cluster.ClusterState.builder;
/**
@ -54,15 +50,14 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeAct
private final AllocationService allocationService;
private final DynamicSettings dynamicSettings;
private final ClusterSettings clusterSettings;
@Inject
public TransportClusterUpdateSettingsAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
AllocationService allocationService, @ClusterDynamicSettings DynamicSettings dynamicSettings,
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
AllocationService allocationService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, ClusterSettings clusterSettings) {
super(settings, ClusterUpdateSettingsAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, ClusterUpdateSettingsRequest::new);
this.allocationService = allocationService;
this.dynamicSettings = dynamicSettings;
this.clusterSettings = clusterSettings;
}
@Override
@ -73,8 +68,8 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeAct
@Override
protected ClusterBlockException checkBlock(ClusterUpdateSettingsRequest request, ClusterState state) {
// allow for dedicated changes to the metadata blocks, so we don't block those to allow to "re-enable" it
if ((request.transientSettings().getAsMap().isEmpty() && request.persistentSettings().getAsMap().size() == 1 && request.persistentSettings().get(MetaData.SETTING_READ_ONLY) != null) ||
request.persistentSettings().getAsMap().isEmpty() && request.transientSettings().getAsMap().size() == 1 && request.transientSettings().get(MetaData.SETTING_READ_ONLY) != null) {
if ((request.transientSettings().getAsMap().isEmpty() && request.persistentSettings().getAsMap().size() == 1 && MetaData.SETTING_READ_ONLY_SETTING.exists(request.persistentSettings())) ||
request.persistentSettings().getAsMap().isEmpty() && request.transientSettings().getAsMap().size() == 1 && MetaData.SETTING_READ_ONLY_SETTING.exists(request.transientSettings())) {
return null;
}
return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
@ -88,9 +83,7 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeAct
@Override
protected void masterOperation(final ClusterUpdateSettingsRequest request, final ClusterState state, final ActionListener<ClusterUpdateSettingsResponse> listener) {
final Settings.Builder transientUpdates = Settings.settingsBuilder();
final Settings.Builder persistentUpdates = Settings.settingsBuilder();
final SettingsUpdater updater = new SettingsUpdater(clusterSettings);
clusterService.submitStateUpdateTask("cluster_update_settings",
new AckedClusterStateUpdateTask<ClusterUpdateSettingsResponse>(Priority.IMMEDIATE, request, listener) {
@ -98,7 +91,7 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeAct
@Override
protected ClusterUpdateSettingsResponse newResponse(boolean acknowledged) {
return new ClusterUpdateSettingsResponse(acknowledged, transientUpdates.build(), persistentUpdates.build());
return new ClusterUpdateSettingsResponse(acknowledged, updater.getTransientUpdates(), updater.getPersistentUpdate());
}
@Override
@ -125,7 +118,7 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeAct
// so we should *not* execute the reroute.
if (!clusterService.state().nodes().localNodeMaster()) {
logger.debug("Skipping reroute after cluster update settings, because node is no longer master");
listener.onResponse(new ClusterUpdateSettingsResponse(updateSettingsAcked, transientUpdates.build(), persistentUpdates.build()));
listener.onResponse(new ClusterUpdateSettingsResponse(updateSettingsAcked, updater.getTransientUpdates(), updater.getPersistentUpdate()));
return;
}
@ -145,13 +138,13 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeAct
@Override
//we return when the cluster reroute is acked or it times out but the acknowledged flag depends on whether the update settings was acknowledged
protected ClusterUpdateSettingsResponse newResponse(boolean acknowledged) {
return new ClusterUpdateSettingsResponse(updateSettingsAcked && acknowledged, transientUpdates.build(), persistentUpdates.build());
return new ClusterUpdateSettingsResponse(updateSettingsAcked && acknowledged, updater.getTransientUpdates(), updater.getPersistentUpdate());
}
@Override
public void onNoLongerMaster(String source) {
logger.debug("failed to preform reroute after cluster settings were updated - current node is no longer a master");
listener.onResponse(new ClusterUpdateSettingsResponse(updateSettingsAcked, transientUpdates.build(), persistentUpdates.build()));
listener.onResponse(new ClusterUpdateSettingsResponse(updateSettingsAcked, updater.getTransientUpdates(), updater.getPersistentUpdate()));
}
@Override
@ -181,58 +174,11 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeAct
@Override
public ClusterState execute(final ClusterState currentState) {
Settings.Builder transientSettings = Settings.settingsBuilder();
transientSettings.put(currentState.metaData().transientSettings());
for (Map.Entry<String, String> entry : request.transientSettings().getAsMap().entrySet()) {
if (dynamicSettings.isDynamicOrLoggingSetting(entry.getKey())) {
String error = dynamicSettings.validateDynamicSetting(entry.getKey(), entry.getValue(), clusterService.state());
if (error == null) {
transientSettings.put(entry.getKey(), entry.getValue());
transientUpdates.put(entry.getKey(), entry.getValue());
changed = true;
} else {
logger.warn("ignoring transient setting [{}], [{}]", entry.getKey(), error);
}
} else {
logger.warn("ignoring transient setting [{}], not dynamically updateable", entry.getKey());
}
}
Settings.Builder persistentSettings = Settings.settingsBuilder();
persistentSettings.put(currentState.metaData().persistentSettings());
for (Map.Entry<String, String> entry : request.persistentSettings().getAsMap().entrySet()) {
if (dynamicSettings.isDynamicOrLoggingSetting(entry.getKey())) {
String error = dynamicSettings.validateDynamicSetting(entry.getKey(), entry.getValue(), clusterService.state());
if (error == null) {
persistentSettings.put(entry.getKey(), entry.getValue());
persistentUpdates.put(entry.getKey(), entry.getValue());
changed = true;
} else {
logger.warn("ignoring persistent setting [{}], [{}]", entry.getKey(), error);
}
} else {
logger.warn("ignoring persistent setting [{}], not dynamically updateable", entry.getKey());
}
}
if (!changed) {
return currentState;
}
MetaData.Builder metaData = MetaData.builder(currentState.metaData())
.persistentSettings(persistentSettings.build())
.transientSettings(transientSettings.build());
ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
boolean updatedReadOnly = metaData.persistentSettings().getAsBoolean(MetaData.SETTING_READ_ONLY, false) || metaData.transientSettings().getAsBoolean(MetaData.SETTING_READ_ONLY, false);
if (updatedReadOnly) {
blocks.addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK);
} else {
blocks.removeGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK);
}
return builder(currentState).metaData(metaData).blocks(blocks).build();
ClusterState clusterState = updater.updateSettings(currentState, request.transientSettings(), request.persistentSettings());
changed = clusterState != currentState;
return clusterState;
}
});
}
}

4
core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/TransportRenderSearchTemplateAction.java
View File

@ -33,6 +33,8 @@ import org.elasticsearch.script.ScriptService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.util.Collections;
public class TransportRenderSearchTemplateAction extends HandledTransportAction<RenderSearchTemplateRequest, RenderSearchTemplateResponse> {
private final ScriptService scriptService;
@ -55,7 +57,7 @@ public class TransportRenderSearchTemplateAction extends HandledTransportAction<
@Override
protected void doRun() throws Exception {
ExecutableScript executable = scriptService.executable(request.template(), ScriptContext.Standard.SEARCH, request);
ExecutableScript executable = scriptService.executable(request.template(), ScriptContext.Standard.SEARCH, request, Collections.emptyMap());
BytesReference processedTemplate = (BytesReference) executable.run();
RenderSearchTemplateResponse response = new RenderSearchTemplateResponse();
response.source(processedTemplate);

28
core/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java
View File

@ -31,31 +31,36 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.settings.NodeSettingsService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
/**
* Close index action
*/
public class TransportCloseIndexAction extends TransportMasterNodeAction<CloseIndexRequest, CloseIndexResponse> implements NodeSettingsService.Listener {
public class TransportCloseIndexAction extends TransportMasterNodeAction<CloseIndexRequest, CloseIndexResponse> {
private final MetaDataIndexStateService indexStateService;
private final DestructiveOperations destructiveOperations;
private volatile boolean closeIndexEnabled;
public static final String SETTING_CLUSTER_INDICES_CLOSE_ENABLE = "cluster.indices.close.enable";
public static final Setting<Boolean> CLUSTER_INDICES_CLOSE_ENABLE_SETTING = Setting.boolSetting("cluster.indices.close.enable", true, true, Setting.Scope.CLUSTER);
@Inject
public TransportCloseIndexAction(Settings settings, TransportService transportService, ClusterService clusterService,
ThreadPool threadPool, MetaDataIndexStateService indexStateService,
NodeSettingsService nodeSettingsService, ActionFilters actionFilters,
ClusterSettings clusterSettings, ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver, DestructiveOperations destructiveOperations) {
super(settings, CloseIndexAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, CloseIndexRequest::new);
this.indexStateService = indexStateService;
this.destructiveOperations = destructiveOperations;
this.closeIndexEnabled = settings.getAsBoolean(SETTING_CLUSTER_INDICES_CLOSE_ENABLE, true);
nodeSettingsService.addListener(this);
this.closeIndexEnabled = CLUSTER_INDICES_CLOSE_ENABLE_SETTING.get(settings);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_INDICES_CLOSE_ENABLE_SETTING, this::setCloseIndexEnabled);
}
private void setCloseIndexEnabled(boolean closeIndexEnabled) {
this.closeIndexEnabled = closeIndexEnabled;
}
@Override
@ -73,7 +78,7 @@ public class TransportCloseIndexAction extends TransportMasterNodeAction<CloseIn
protected void doExecute(CloseIndexRequest request, ActionListener<CloseIndexResponse> listener) {
destructiveOperations.failDestructive(request.indices());
if (closeIndexEnabled == false) {
throw new IllegalStateException("closing indices is disabled - set [" + SETTING_CLUSTER_INDICES_CLOSE_ENABLE + ": true] to enable it. NOTE: closed indices still consume a significant amount of diskspace");
throw new IllegalStateException("closing indices is disabled - set [" + CLUSTER_INDICES_CLOSE_ENABLE_SETTING.getKey() + ": true] to enable it. NOTE: closed indices still consume a significant amount of diskspace");
}
super.doExecute(request, listener);
}
@ -104,13 +109,4 @@ public class TransportCloseIndexAction extends TransportMasterNodeAction<CloseIn
}
});
}
@Override
public void onRefreshSettings(Settings settings) {
final boolean enable = settings.getAsBoolean(SETTING_CLUSTER_INDICES_CLOSE_ENABLE, this.closeIndexEnabled);
if (enable != this.closeIndexEnabled) {
logger.info("updating [{}] from [{}] to [{}]", SETTING_CLUSTER_INDICES_CLOSE_ENABLE, this.closeIndexEnabled, enable);
this.closeIndexEnabled = enable;
}
}
}

4
core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
View File

@ -31,7 +31,6 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.MetaDataDeleteIndexService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.settings.NodeSettingsService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
@ -45,8 +44,7 @@ public class TransportDeleteIndexAction extends TransportMasterNodeAction<Delete
@Inject
public TransportDeleteIndexAction(Settings settings, TransportService transportService, ClusterService clusterService,
ThreadPool threadPool, MetaDataDeleteIndexService deleteIndexService,
NodeSettingsService nodeSettingsService, ActionFilters actionFilters,
ThreadPool threadPool, MetaDataDeleteIndexService deleteIndexService, ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver, DestructiveOperations destructiveOperations) {
super(settings, DeleteIndexAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, DeleteIndexRequest::new);
this.deleteIndexService = deleteIndexService;

34
core/src/main/java/org/apache/lucene/queryparser/classic/MissingFieldQueryExtension.java → core/src/main/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushAction.java
View File

@ -17,26 +17,28 @@
* under the License.
*/
package org.apache.lucene.queryparser.classic;
package org.elasticsearch.action.admin.indices.flush;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.Query;
import org.elasticsearch.index.query.MissingQueryBuilder;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.action.Action;
import org.elasticsearch.client.ElasticsearchClient;
/**
*
*/
public class MissingFieldQueryExtension implements FieldQueryExtension {
public static final String NAME = "_missing_";
public class SyncedFlushAction extends Action<SyncedFlushRequest, SyncedFlushResponse, SyncedFlushRequestBuilder> {
public static final SyncedFlushAction INSTANCE = new SyncedFlushAction();
public static final String NAME = "indices:admin/synced_flush";
private SyncedFlushAction() {
super(NAME);
}
@Override
public Query query(QueryShardContext context, String queryText) {
Query query = MissingQueryBuilder.newFilter(context, queryText, MissingQueryBuilder.DEFAULT_EXISTENCE_VALUE, MissingQueryBuilder.DEFAULT_NULL_VALUE);
if (query != null) {
return new ConstantScoreQuery(query);
}
return null;
public SyncedFlushResponse newResponse() {
return new SyncedFlushResponse();
}
@Override
public SyncedFlushRequestBuilder newRequestBuilder(ElasticsearchClient client) {
return new SyncedFlushRequestBuilder(client, this);
}
}

64
core/src/main/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushRequest.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.admin.indices.flush;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.support.broadcast.BroadcastRequest;
import java.util.Arrays;
/**
* A synced flush request to sync flush one or more indices. The synced flush process of an index performs a flush
* and writes the same sync id to primary and all copies.
*
* <p>Best created with {@link org.elasticsearch.client.Requests#syncedFlushRequest(String...)}. </p>
*
* @see org.elasticsearch.client.Requests#flushRequest(String...)
* @see org.elasticsearch.client.IndicesAdminClient#syncedFlush(SyncedFlushRequest)
* @see SyncedFlushResponse
*/
public class SyncedFlushRequest extends BroadcastRequest<SyncedFlushRequest> {
public SyncedFlushRequest() {
}
/**
* Copy constructor that creates a new synced flush request that is a copy of the one provided as an argument.
* The new request will inherit though headers and context from the original request that caused it.
*/
public SyncedFlushRequest(ActionRequest originalRequest) {
super(originalRequest);
}
/**
* Constructs a new synced flush request against one or more indices. If nothing is provided, all indices will
* be sync flushed.
*/
public SyncedFlushRequest(String... indices) {
super(indices);
}
@Override
public String toString() {
return "SyncedFlushRequest{" +
"indices=" + Arrays.toString(indices) + "}";
}
}

36
core/src/main/java/org/elasticsearch/rest/RestModule.java → core/src/main/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushRequestBuilder.java
View File

@ -17,35 +17,25 @@
* under the License.
*/
package org.elasticsearch.rest;
package org.elasticsearch.action.admin.indices.flush;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.rest.action.RestActionModule;
import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.client.ElasticsearchClient;
import java.util.ArrayList;
import java.util.List;
public class SyncedFlushRequestBuilder extends ActionRequestBuilder<SyncedFlushRequest, SyncedFlushResponse, SyncedFlushRequestBuilder> {
/**
*
*/
public class RestModule extends AbstractModule {
private final Settings settings;
private List<Class<? extends BaseRestHandler>> restPluginsActions = new ArrayList<>();
public void addRestAction(Class<? extends BaseRestHandler> restAction) {
restPluginsActions.add(restAction);
public SyncedFlushRequestBuilder(ElasticsearchClient client, SyncedFlushAction action) {
super(client, action, new SyncedFlushRequest());
}
public RestModule(Settings settings) {
this.settings = settings;
public SyncedFlushRequestBuilder setIndices(String[] indices) {
super.request().indices(indices);
return this;
}
@Override
protected void configure() {
bind(RestController.class).asEagerSingleton();
new RestActionModule(restPluginsActions).configure(binder());
public SyncedFlushRequestBuilder setIndicesOptions(IndicesOptions indicesOptions) {
super.request().indicesOptions(indicesOptions);
return this;
}
}

97
core/src/main/java/org/elasticsearch/indices/flush/IndicesSyncedFlushResult.java → core/src/main/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushResponse.java
View File

@ -16,16 +16,25 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.indices.flush;
package org.elasticsearch.action.admin.indices.flush;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.util.iterable.Iterables;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
import org.elasticsearch.indices.flush.ShardsSyncedFlushResult;
import org.elasticsearch.indices.flush.SyncedFlushService;
import org.elasticsearch.rest.RestStatus;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -34,13 +43,16 @@ import static java.util.Collections.unmodifiableMap;
/**
* The result of performing a sync flush operation on all shards of multiple indices
*/
public class IndicesSyncedFlushResult implements ToXContent {
public class SyncedFlushResponse extends ActionResponse implements ToXContent {
final Map<String, List<ShardsSyncedFlushResult>> shardsResultPerIndex;
final ShardCounts shardCounts;
Map<String, List<ShardsSyncedFlushResult>> shardsResultPerIndex;
ShardCounts shardCounts;
SyncedFlushResponse() {
public IndicesSyncedFlushResult(Map<String, List<ShardsSyncedFlushResult>> shardsResultPerIndex) {
}
public SyncedFlushResponse(Map<String, List<ShardsSyncedFlushResult>> shardsResultPerIndex) {
// shardsResultPerIndex is never modified after it is passed to this
// constructor so this is safe even though shardsResultPerIndex is a
// ConcurrentHashMap
@ -48,17 +60,23 @@ public class IndicesSyncedFlushResult implements ToXContent {
this.shardCounts = calculateShardCounts(Iterables.flatten(shardsResultPerIndex.values()));
}
/** total number shards, including replicas, both assigned and unassigned */
/**
* total number shards, including replicas, both assigned and unassigned
*/
public int totalShards() {
return shardCounts.total;
}
/** total number of shards for which the operation failed */
/**
* total number of shards for which the operation failed
*/
public int failedShards() {
return shardCounts.failed;
}
/** total number of shards which were successfully sync-flushed */
/**
* total number of shards which were successfully sync-flushed
*/
public int successfulShards() {
return shardCounts.successful;
}
@ -91,8 +109,8 @@ public class IndicesSyncedFlushResult implements ToXContent {
builder.endObject();
continue;
}
Map<ShardRouting, SyncedFlushService.SyncedFlushResponse> failedShards = shardResults.failedShards();
for (Map.Entry<ShardRouting, SyncedFlushService.SyncedFlushResponse> shardEntry : failedShards.entrySet()) {
Map<ShardRouting, SyncedFlushService.ShardSyncedFlushResponse> failedShards = shardResults.failedShards();
for (Map.Entry<ShardRouting, SyncedFlushService.ShardSyncedFlushResponse> shardEntry : failedShards.entrySet()) {
builder.startObject();
builder.field(Fields.SHARD, shardResults.shardId().id());
builder.field(Fields.REASON, shardEntry.getValue().failureReason());
@ -123,11 +141,11 @@ public class IndicesSyncedFlushResult implements ToXContent {
return new ShardCounts(total, successful, failed);
}
static final class ShardCounts implements ToXContent {
static final class ShardCounts implements ToXContent, Streamable {
public final int total;
public final int successful;
public final int failed;
public int total;
public int successful;
public int failed;
ShardCounts(int total, int successful, int failed) {
this.total = total;
@ -135,6 +153,10 @@ public class IndicesSyncedFlushResult implements ToXContent {
this.failed = failed;
}
ShardCounts() {
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(Fields.TOTAL, total);
@ -142,6 +164,20 @@ public class IndicesSyncedFlushResult implements ToXContent {
builder.field(Fields.FAILED, failed);
return builder;
}
@Override
public void readFrom(StreamInput in) throws IOException {
total = in.readInt();
successful = in.readInt();
failed = in.readInt();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeInt(total);
out.writeInt(successful);
out.writeInt(failed);
}
}
static final class Fields {
@ -154,4 +190,37 @@ public class IndicesSyncedFlushResult implements ToXContent {
static final XContentBuilderString ROUTING = new XContentBuilderString("routing");
static final XContentBuilderString REASON = new XContentBuilderString("reason");
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
shardCounts = new ShardCounts();
shardCounts.readFrom(in);
Map<String, List<ShardsSyncedFlushResult>> tmpShardsResultPerIndex = new HashMap<>();
int numShardsResults = in.readInt();
for (int i =0 ; i< numShardsResults; i++) {
String index = in.readString();
List<ShardsSyncedFlushResult> shardsSyncedFlushResults = new ArrayList<>();
int numShards = in.readInt();
for (int j =0; j< numShards; j++) {
shardsSyncedFlushResults.add(ShardsSyncedFlushResult.readShardsSyncedFlushResult(in));
}
tmpShardsResultPerIndex.put(index, shardsSyncedFlushResults);
}
shardsResultPerIndex = Collections.unmodifiableMap(tmpShardsResultPerIndex);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
shardCounts.writeTo(out);
out.writeInt(shardsResultPerIndex.size());
for (Map.Entry<String, List<ShardsSyncedFlushResult>> entry : shardsResultPerIndex.entrySet()) {
out.writeString(entry.getKey());
out.writeInt(entry.getValue().size());
for (ShardsSyncedFlushResult shardsSyncedFlushResult : entry.getValue()) {
shardsSyncedFlushResult.writeTo(out);
}
}
}
}

52
core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportSyncedFlushAction.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.admin.indices.flush;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.HandledTransportAction;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.indices.flush.SyncedFlushService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
/**
* Synced flush Action.
*/
public class TransportSyncedFlushAction extends HandledTransportAction<SyncedFlushRequest, SyncedFlushResponse> {
SyncedFlushService syncedFlushService;
@Inject
public TransportSyncedFlushAction(Settings settings, ThreadPool threadPool,
TransportService transportService, ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver,
SyncedFlushService syncedFlushService) {
super(settings, SyncedFlushAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, SyncedFlushRequest::new);
this.syncedFlushService = syncedFlushService;
}
@Override
protected void doExecute(SyncedFlushRequest request, ActionListener<SyncedFlushResponse> listener) {
syncedFlushService.attemptSyncedFlush(request.indices(), request.indicesOptions(), listener);
}
}

5
core/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java
View File

@ -32,12 +32,9 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.settings.NodeSettingsService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.util.Arrays;
/**
* Open index action
*/
@ -49,7 +46,7 @@ public class TransportOpenIndexAction extends TransportMasterNodeAction<OpenInde
@Inject
public TransportOpenIndexAction(Settings settings, TransportService transportService, ClusterService clusterService,
ThreadPool threadPool, MetaDataIndexStateService indexStateService,
NodeSettingsService nodeSettingsService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
DestructiveOperations destructiveOperations) {
super(settings, OpenIndexAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, OpenIndexRequest::new);
this.indexStateService = indexStateService;

55
core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
View File

@ -56,13 +56,14 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
public static class StoreStatus implements Streamable, ToXContent, Comparable<StoreStatus> {
private DiscoveryNode node;
private long version;
private String allocationId;
private Throwable storeException;
private Allocation allocation;
private AllocationStatus allocationStatus;
/**
* The status of the shard store with respect to the cluster
*/
public enum Allocation {
public enum AllocationStatus {
/**
* Allocated as primary
@ -81,16 +82,16 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
private final byte id;
Allocation(byte id) {
AllocationStatus(byte id) {
this.id = id;
}
private static Allocation fromId(byte id) {
private static AllocationStatus fromId(byte id) {
switch (id) {
case 0: return PRIMARY;
case 1: return REPLICA;
case 2: return UNUSED;
default: throw new IllegalArgumentException("unknown id for allocation [" + id + "]");
default: throw new IllegalArgumentException("unknown id for allocation status [" + id + "]");
}
}
@ -99,11 +100,11 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
case 0: return "primary";
case 1: return "replica";
case 2: return "unused";
default: throw new IllegalArgumentException("unknown id for allocation [" + id + "]");
default: throw new IllegalArgumentException("unknown id for allocation status [" + id + "]");
}
}
private static Allocation readFrom(StreamInput in) throws IOException {
private static AllocationStatus readFrom(StreamInput in) throws IOException {
return fromId(in.readByte());
}
@ -115,10 +116,11 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
private StoreStatus() {
}
public StoreStatus(DiscoveryNode node, long version, Allocation allocation, Throwable storeException) {
public StoreStatus(DiscoveryNode node, long version, String allocationId, AllocationStatus allocationStatus, Throwable storeException) {
this.node = node;
this.version = version;
this.allocation = allocation;
this.allocationId = allocationId;
this.allocationStatus = allocationStatus;
this.storeException = storeException;
}
@ -130,13 +132,20 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
}
/**
* Version of the store, used to select the store that will be
* used as a primary.
* Version of the store
*/
public long getVersion() {
return version;
}
/**
* AllocationStatus id of the store, used to select the store that will be
* used as a primary.
*/
public String getAllocationId() {
return allocationId;
}
/**
* Exception while trying to open the
* shard index or from when the shard failed
@ -146,13 +155,13 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
}
/**
* The allocation status of the store.
* {@link Allocation#PRIMARY} indicates a primary shard copy
* {@link Allocation#REPLICA} indicates a replica shard copy
* {@link Allocation#UNUSED} indicates an unused shard copy
* The allocationStatus status of the store.
* {@link AllocationStatus#PRIMARY} indicates a primary shard copy
* {@link AllocationStatus#REPLICA} indicates a replica shard copy
* {@link AllocationStatus#UNUSED} indicates an unused shard copy
*/
public Allocation getAllocation() {
return allocation;
public AllocationStatus getAllocationStatus() {
return allocationStatus;
}
static StoreStatus readStoreStatus(StreamInput in) throws IOException {
@ -165,7 +174,8 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
public void readFrom(StreamInput in) throws IOException {
node = DiscoveryNode.readNode(in);
version = in.readLong();
allocation = Allocation.readFrom(in);
allocationId = in.readOptionalString();
allocationStatus = AllocationStatus.readFrom(in);
if (in.readBoolean()) {
storeException = in.readThrowable();
}
@ -175,7 +185,8 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
public void writeTo(StreamOutput out) throws IOException {
node.writeTo(out);
out.writeLong(version);
allocation.writeTo(out);
out.writeOptionalString(allocationId);
allocationStatus.writeTo(out);
if (storeException != null) {
out.writeBoolean(true);
out.writeThrowable(storeException);
@ -188,7 +199,8 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
node.toXContent(builder, params);
builder.field(Fields.VERSION, version);
builder.field(Fields.ALLOCATED, allocation.value());
builder.field(Fields.ALLOCATION_ID, allocationId);
builder.field(Fields.ALLOCATED, allocationStatus.value());
if (storeException != null) {
builder.startObject(Fields.STORE_EXCEPTION);
ElasticsearchException.toXContent(builder, params, storeException);
@ -206,7 +218,7 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
} else {
int compare = Long.compare(other.version, version);
if (compare == 0) {
return Integer.compare(allocation.id, other.allocation.id);
return Integer.compare(allocationStatus.id, other.allocationStatus.id);
}
return compare;
}
@ -379,6 +391,7 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
static final XContentBuilderString STORES = new XContentBuilderString("stores");
// StoreStatus fields
static final XContentBuilderString VERSION = new XContentBuilderString("version");
static final XContentBuilderString ALLOCATION_ID = new XContentBuilderString("allocation_id");
static final XContentBuilderString STORE_EXCEPTION = new XContentBuilderString("store_exception");
static final XContentBuilderString ALLOCATED = new XContentBuilderString("allocation");
}

16
core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
View File

@ -179,8 +179,8 @@ public class TransportIndicesShardStoresAction extends TransportMasterNodeReadAc
}
for (NodeGatewayStartedShards response : fetchResponse.responses) {
if (shardExistsInNode(response)) {
IndicesShardStoresResponse.StoreStatus.Allocation allocation = getAllocation(fetchResponse.shardId.getIndex(), fetchResponse.shardId.id(), response.getNode());
storeStatuses.add(new IndicesShardStoresResponse.StoreStatus(response.getNode(), response.version(), allocation, response.storeException()));
IndicesShardStoresResponse.StoreStatus.AllocationStatus allocationStatus = getAllocationStatus(fetchResponse.shardId.getIndex(), fetchResponse.shardId.id(), response.getNode());
storeStatuses.add(new IndicesShardStoresResponse.StoreStatus(response.getNode(), response.version(), response.allocationId(), allocationStatus, response.storeException()));
}
}
CollectionUtil.timSort(storeStatuses);
@ -193,27 +193,27 @@ public class TransportIndicesShardStoresAction extends TransportMasterNodeReadAc
listener.onResponse(new IndicesShardStoresResponse(indicesStoreStatusesBuilder.build(), Collections.unmodifiableList(failureBuilder)));
}
private IndicesShardStoresResponse.StoreStatus.Allocation getAllocation(String index, int shardID, DiscoveryNode node) {
private IndicesShardStoresResponse.StoreStatus.AllocationStatus getAllocationStatus(String index, int shardID, DiscoveryNode node) {
for (ShardRouting shardRouting : routingNodes.node(node.id())) {
ShardId shardId = shardRouting.shardId();
if (shardId.id() == shardID && shardId.getIndex().equals(index)) {
if (shardRouting.primary()) {
return IndicesShardStoresResponse.StoreStatus.Allocation.PRIMARY;
return IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY;
} else if (shardRouting.assignedToNode()) {
return IndicesShardStoresResponse.StoreStatus.Allocation.REPLICA;
return IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA;
} else {
return IndicesShardStoresResponse.StoreStatus.Allocation.UNUSED;
return IndicesShardStoresResponse.StoreStatus.AllocationStatus.UNUSED;
}
}
}
return IndicesShardStoresResponse.StoreStatus.Allocation.UNUSED;
return IndicesShardStoresResponse.StoreStatus.AllocationStatus.UNUSED;
}
/**
* A shard exists/existed in a node only if shard state file exists in the node
*/
private boolean shardExistsInNode(final NodeGatewayStartedShards response) {
return response.storeException() != null || response.version() != -1;
return response.storeException() != null || response.version() != -1 || response.allocationId() != null;
}
@Override

203
core/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java Normal file
View File

@ -0,0 +1,203 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.bulk;
import org.elasticsearch.common.unit.TimeValue;
import java.util.Iterator;
import java.util.NoSuchElementException;
/**
* Provides a backoff policy for bulk requests. Whenever a bulk request is rejected due to resource constraints (i.e. the client's internal
* thread pool is full), the backoff policy decides how long the bulk processor will wait before the operation is retried internally.
*
* Notes for implementing custom subclasses:
*
* The underlying mathematical principle of <code>BackoffPolicy</code> are progressions which can be either finite or infinite although
* the latter should not be used for retrying. A progression can be mapped to a <code>java.util.Iterator</code> with the following
* semantics:
*
* <ul>
* <li><code>#hasNext()</code> determines whether the progression has more elements. Return <code>true</code> for infinite progressions</li>
* <li><code>#next()</code> determines the next element in the progression, i.e. the next wait time period</li>
* </ul>
*
* Note that backoff policies are exposed as <code>Iterables</code> in order to be consumed multiple times.
*/
public abstract class BackoffPolicy implements Iterable<TimeValue> {
private static final BackoffPolicy NO_BACKOFF = new NoBackoff();
/**
* Creates a backoff policy that will not allow any backoff, i.e. an operation will fail after the first attempt.
*
* @return A backoff policy without any backoff period. The returned instance is thread safe.
*/
public static BackoffPolicy noBackoff() {
return NO_BACKOFF;
}
/**
* Creates an new constant backoff policy with the provided configuration.
*
* @param delay The delay defines how long to wait between retry attempts. Must not be null.
* Must be &lt;= <code>Integer.MAX_VALUE</code> ms.
* @param maxNumberOfRetries The maximum number of retries. Must be a non-negative number.
* @return A backoff policy with a constant wait time between retries. The returned instance is thread safe but each
* iterator created from it should only be used by a single thread.
*/
public static BackoffPolicy constantBackoff(TimeValue delay, int maxNumberOfRetries) {
return new ConstantBackoff(checkDelay(delay), maxNumberOfRetries);
}
/**
* Creates an new exponential backoff policy with a default configuration of 50 ms initial wait period and 8 retries taking
* roughly 5.1 seconds in total.
*
* @return A backoff policy with an exponential increase in wait time for retries. The returned instance is thread safe but each
* iterator created from it should only be used by a single thread.
*/
public static BackoffPolicy exponentialBackoff() {
return exponentialBackoff(TimeValue.timeValueMillis(50), 8);
}
/**
* Creates an new exponential backoff policy with the provided configuration.
*
* @param initialDelay The initial delay defines how long to wait for the first retry attempt. Must not be null.
* Must be &lt;= <code>Integer.MAX_VALUE</code> ms.
* @param maxNumberOfRetries The maximum number of retries. Must be a non-negative number.
* @return A backoff policy with an exponential increase in wait time for retries. The returned instance is thread safe but each
* iterator created from it should only be used by a single thread.
*/
public static BackoffPolicy exponentialBackoff(TimeValue initialDelay, int maxNumberOfRetries) {
return new ExponentialBackoff((int) checkDelay(initialDelay).millis(), maxNumberOfRetries);
}
private static TimeValue checkDelay(TimeValue delay) {
if (delay.millis() > Integer.MAX_VALUE) {
throw new IllegalArgumentException("delay must be <= " + Integer.MAX_VALUE + " ms");
}
return delay;
}
private static class NoBackoff extends BackoffPolicy {
@Override
public Iterator<TimeValue> iterator() {
return new Iterator<TimeValue>() {
@Override
public boolean hasNext() {
return false;
}
@Override
public TimeValue next() {
throw new NoSuchElementException("No backoff");
}
};
}
}
private static class ExponentialBackoff extends BackoffPolicy {
private final int start;
private final int numberOfElements;
private ExponentialBackoff(int start, int numberOfElements) {
assert start >= 0;
assert numberOfElements >= 0;
this.start = start;
this.numberOfElements = numberOfElements;
}
@Override
public Iterator<TimeValue> iterator() {
return new ExponentialBackoffIterator(start, numberOfElements);
}
}
private static class ExponentialBackoffIterator implements Iterator<TimeValue> {
private final int numberOfElements;
private final int start;
private int currentlyConsumed;
private ExponentialBackoffIterator(int start, int numberOfElements) {
this.start = start;
this.numberOfElements = numberOfElements;
}
@Override
public boolean hasNext() {
return currentlyConsumed < numberOfElements;
}
@Override
public TimeValue next() {
if (!hasNext()) {
throw new NoSuchElementException("Only up to " + numberOfElements + " elements");
}
int result = start + 10 * ((int) Math.exp(0.8d * (currentlyConsumed)) - 1);
currentlyConsumed++;
return TimeValue.timeValueMillis(result);
}
}
private static final class ConstantBackoff extends BackoffPolicy {
private final TimeValue delay;
private final int numberOfElements;
public ConstantBackoff(TimeValue delay, int numberOfElements) {
assert numberOfElements >= 0;
this.delay = delay;
this.numberOfElements = numberOfElements;
}
@Override
public Iterator<TimeValue> iterator() {
return new ConstantBackoffIterator(delay, numberOfElements);
}
}
private static final class ConstantBackoffIterator implements Iterator<TimeValue> {
private final TimeValue delay;
private final int numberOfElements;
private int curr;
public ConstantBackoffIterator(TimeValue delay, int numberOfElements) {
this.delay = delay;
this.numberOfElements = numberOfElements;
}
@Override
public boolean hasNext() {
return curr < numberOfElements;
}
@Override
public TimeValue next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
curr++;
return delay;
}
}
}

108
core/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.action.bulk;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
@ -48,7 +47,7 @@ public class BulkProcessor implements Closeable {
/**
* A listener for the execution.
*/
public static interface Listener {
public interface Listener {
/**
* Callback before the bulk is executed.
@ -62,6 +61,9 @@ public class BulkProcessor implements Closeable {
/**
* Callback after a failed execution of bulk request.
*
* Note that in case an instance of <code>InterruptedException</code> is passed, which means that request processing has been
* cancelled externally, the thread's interruption status has been restored prior to calling this method.
*/
void afterBulk(long executionId, BulkRequest request, Throwable failure);
}
@ -79,6 +81,7 @@ public class BulkProcessor implements Closeable {
private int bulkActions = 1000;
private ByteSizeValue bulkSize = new ByteSizeValue(5, ByteSizeUnit.MB);
private TimeValue flushInterval = null;
private BackoffPolicy backoffPolicy = BackoffPolicy.exponentialBackoff();
/**
* Creates a builder of bulk processor with the client to use and the listener that will be used
@ -136,11 +139,27 @@ public class BulkProcessor implements Closeable {
return this;
}
/**
* Sets a custom backoff policy. The backoff policy defines how the bulk processor should handle retries of bulk requests internally
* in case they have failed due to resource constraints (i.e. a thread pool was full).
*
* The default is to back off exponentially.
*
* @see org.elasticsearch.action.bulk.BackoffPolicy#exponentialBackoff()
*/
public Builder setBackoffPolicy(BackoffPolicy backoffPolicy) {
if (backoffPolicy == null) {
throw new NullPointerException("'backoffPolicy' must not be null. To disable backoff, pass BackoffPolicy.noBackoff()");
}
this.backoffPolicy = backoffPolicy;
return this;
}
/**
* Builds a new bulk processor.
*/
public BulkProcessor build() {
return new BulkProcessor(client, listener, name, concurrentRequests, bulkActions, bulkSize, flushInterval);
return new BulkProcessor(client, backoffPolicy, listener, name, concurrentRequests, bulkActions, bulkSize, flushInterval);
}
}
@ -148,42 +167,31 @@ public class BulkProcessor implements Closeable {
if (client == null) {
throw new NullPointerException("The client you specified while building a BulkProcessor is null");
}
return new Builder(client, listener);
}
private final Client client;
private final Listener listener;
private final String name;
private final int concurrentRequests;
private final int bulkActions;
private final long bulkSize;
private final TimeValue flushInterval;
private final Semaphore semaphore;
private final ScheduledThreadPoolExecutor scheduler;
private final ScheduledFuture scheduledFuture;
private final AtomicLong executionIdGen = new AtomicLong();
private BulkRequest bulkRequest;
private final BulkRequestHandler bulkRequestHandler;
private volatile boolean closed = false;
BulkProcessor(Client client, Listener listener, @Nullable String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, @Nullable TimeValue flushInterval) {
this.client = client;
this.listener = listener;
this.name = name;
this.concurrentRequests = concurrentRequests;
BulkProcessor(Client client, BackoffPolicy backoffPolicy, Listener listener, @Nullable String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, @Nullable TimeValue flushInterval) {
this.bulkActions = bulkActions;
this.bulkSize = bulkSize.bytes();
this.semaphore = new Semaphore(concurrentRequests);
this.bulkRequest = new BulkRequest();
this.bulkRequestHandler = (concurrentRequests == 0) ? BulkRequestHandler.syncHandler(client, backoffPolicy, listener) : BulkRequestHandler.asyncHandler(client, backoffPolicy, listener, concurrentRequests);
this.flushInterval = flushInterval;
if (flushInterval != null) {
this.scheduler = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1, EsExecutors.daemonThreadFactory(client.settings(), (name != null ? "[" + name + "]" : "") + "bulk_processor"));
this.scheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
@ -231,14 +239,7 @@ public class BulkProcessor implements Closeable {
if (bulkRequest.numberOfActions() > 0) {
execute();
}
if (this.concurrentRequests < 1) {
return true;
}
if (semaphore.tryAcquire(this.concurrentRequests, timeout, unit)) {
semaphore.release(this.concurrentRequests);
return true;
}
return false;
return this.bulkRequestHandler.awaitClose(timeout, unit);
}
/**
@ -308,58 +309,7 @@ public class BulkProcessor implements Closeable {
final long executionId = executionIdGen.incrementAndGet();
this.bulkRequest = new BulkRequest();
if (concurrentRequests == 0) {
// execute in a blocking fashion...
boolean afterCalled = false;
try {
listener.beforeBulk(executionId, bulkRequest);
BulkResponse bulkItemResponses = client.bulk(bulkRequest).actionGet();
afterCalled = true;
listener.afterBulk(executionId, bulkRequest, bulkItemResponses);
} catch (Exception e) {
if (!afterCalled) {
listener.afterBulk(executionId, bulkRequest, e);
}
}
} else {
boolean success = false;
boolean acquired = false;
try {
listener.beforeBulk(executionId, bulkRequest);
semaphore.acquire();
acquired = true;
client.bulk(bulkRequest, new ActionListener<BulkResponse>() {
@Override
public void onResponse(BulkResponse response) {
try {
listener.afterBulk(executionId, bulkRequest, response);
} finally {
semaphore.release();
}
}
@Override
public void onFailure(Throwable e) {
try {
listener.afterBulk(executionId, bulkRequest, e);
} finally {
semaphore.release();
}
}
});
success = true;
} catch (InterruptedException e) {
Thread.interrupted();
listener.afterBulk(executionId, bulkRequest, e);
} catch (Throwable t) {
listener.afterBulk(executionId, bulkRequest, t);
} finally {
if (!success && acquired) { // if we fail on client.bulk() release the semaphore
semaphore.release();
}
}
}
this.bulkRequestHandler.execute(bulkRequest, executionId);
}
private boolean isOverTheLimit() {

166
core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java Normal file
View File

@ -0,0 +1,166 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.bulk;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
/**
* Abstracts the low-level details of bulk request handling
*/
abstract class BulkRequestHandler {
protected final ESLogger logger;
protected final Client client;
protected BulkRequestHandler(Client client) {
this.client = client;
this.logger = Loggers.getLogger(getClass(), client.settings());
}
public abstract void execute(BulkRequest bulkRequest, long executionId);
public abstract boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException;
public static BulkRequestHandler syncHandler(Client client, BackoffPolicy backoffPolicy, BulkProcessor.Listener listener) {
return new SyncBulkRequestHandler(client, backoffPolicy, listener);
}
public static BulkRequestHandler asyncHandler(Client client, BackoffPolicy backoffPolicy, BulkProcessor.Listener listener, int concurrentRequests) {
return new AsyncBulkRequestHandler(client, backoffPolicy, listener, concurrentRequests);
}
private static class SyncBulkRequestHandler extends BulkRequestHandler {
private final BulkProcessor.Listener listener;
private final BackoffPolicy backoffPolicy;
public SyncBulkRequestHandler(Client client, BackoffPolicy backoffPolicy, BulkProcessor.Listener listener) {
super(client);
this.backoffPolicy = backoffPolicy;
this.listener = listener;
}
@Override
public void execute(BulkRequest bulkRequest, long executionId) {
boolean afterCalled = false;
try {
listener.beforeBulk(executionId, bulkRequest);
BulkResponse bulkResponse = Retry
.on(EsRejectedExecutionException.class)
.policy(backoffPolicy)
.withSyncBackoff(client, bulkRequest);
afterCalled = true;
listener.afterBulk(executionId, bulkRequest, bulkResponse);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
logger.info("Bulk request {} has been cancelled.", e, executionId);
if (!afterCalled) {
listener.afterBulk(executionId, bulkRequest, e);
}
} catch (Throwable t) {
logger.warn("Failed to execute bulk request {}.", t, executionId);
if (!afterCalled) {
listener.afterBulk(executionId, bulkRequest, t);
}
}
}
@Override
public boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException {
// we are "closed" immediately as there is no request in flight
return true;
}
}
private static class AsyncBulkRequestHandler extends BulkRequestHandler {
private final BackoffPolicy backoffPolicy;
private final BulkProcessor.Listener listener;
private final Semaphore semaphore;
private final int concurrentRequests;
private AsyncBulkRequestHandler(Client client, BackoffPolicy backoffPolicy, BulkProcessor.Listener listener, int concurrentRequests) {
super(client);
this.backoffPolicy = backoffPolicy;
assert concurrentRequests > 0;
this.listener = listener;
this.concurrentRequests = concurrentRequests;
this.semaphore = new Semaphore(concurrentRequests);
}
@Override
public void execute(BulkRequest bulkRequest, long executionId) {
boolean bulkRequestSetupSuccessful = false;
boolean acquired = false;
try {
listener.beforeBulk(executionId, bulkRequest);
semaphore.acquire();
acquired = true;
Retry.on(EsRejectedExecutionException.class)
.policy(backoffPolicy)
.withAsyncBackoff(client, bulkRequest, new ActionListener<BulkResponse>() {
@Override
public void onResponse(BulkResponse response) {
try {
listener.afterBulk(executionId, bulkRequest, response);
} finally {
semaphore.release();
}
}
@Override
public void onFailure(Throwable e) {
try {
listener.afterBulk(executionId, bulkRequest, e);
} finally {
semaphore.release();
}
}
});
bulkRequestSetupSuccessful = true;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
logger.info("Bulk request {} has been cancelled.", e, executionId);
listener.afterBulk(executionId, bulkRequest, e);
} catch (Throwable t) {
logger.warn("Failed to execute bulk request {}.", t, executionId);
listener.afterBulk(executionId, bulkRequest, t);
} finally {
if (!bulkRequestSetupSuccessful && acquired) { // if we fail on client.bulk() release the semaphore
semaphore.release();
}
}
}
@Override
public boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException {
if (semaphore.tryAcquire(this.concurrentRequests, timeout, unit)) {
semaphore.release(this.concurrentRequests);
return true;
}
return false;
}
}
}

237
core/src/main/java/org/elasticsearch/action/bulk/Retry.java Normal file
View File

@ -0,0 +1,237 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.bulk;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.FutureUtils;
import org.elasticsearch.threadpool.ThreadPool;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.*;
import java.util.function.Predicate;
/**
* Encapsulates synchronous and asynchronous retry logic.
*/
class Retry {
private final Class<? extends Throwable> retryOnThrowable;
private BackoffPolicy backoffPolicy;
public static Retry on(Class<? extends Throwable> retryOnThrowable) {
return new Retry(retryOnThrowable);
}
/**
* @param backoffPolicy The backoff policy that defines how long and how often to wait for retries.
*/
public Retry policy(BackoffPolicy backoffPolicy) {
this.backoffPolicy = backoffPolicy;
return this;
}
Retry(Class<? extends Throwable> retryOnThrowable) {
this.retryOnThrowable = retryOnThrowable;
}
/**
* Invokes #bulk(BulkRequest, ActionListener) on the provided client. Backs off on the provided exception and delegates results to the
* provided listener.
*
* @param client Client invoking the bulk request.
* @param bulkRequest The bulk request that should be executed.
* @param listener A listener that is invoked when the bulk request finishes or completes with an exception. The listener is not
*/
public void withAsyncBackoff(Client client, BulkRequest bulkRequest, ActionListener<BulkResponse> listener) {
AsyncRetryHandler r = new AsyncRetryHandler(retryOnThrowable, backoffPolicy, client, listener);
r.execute(bulkRequest);
}
/**
* Invokes #bulk(BulkRequest) on the provided client. Backs off on the provided exception.
*
* @param client Client invoking the bulk request.
* @param bulkRequest The bulk request that should be executed.
* @return the bulk response as returned by the client.
* @throws Exception Any exception thrown by the callable.
*/
public BulkResponse withSyncBackoff(Client client, BulkRequest bulkRequest) throws Exception {
return SyncRetryHandler
.create(retryOnThrowable, backoffPolicy, client)
.executeBlocking(bulkRequest)
.actionGet();
}
static class AbstractRetryHandler implements ActionListener<BulkResponse> {
private final ESLogger logger;
private final Client client;
private final ActionListener<BulkResponse> listener;
private final Iterator<TimeValue> backoff;
private final Class<? extends Throwable> retryOnThrowable;
// Access only when holding a client-side lock, see also #addResponses()
private final List<BulkItemResponse> responses = new ArrayList<>();
private final long startTimestampNanos;
// needed to construct the next bulk request based on the response to the previous one
// volatile as we're called from a scheduled thread
private volatile BulkRequest currentBulkRequest;
private volatile ScheduledFuture<?> scheduledRequestFuture;
public AbstractRetryHandler(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy, Client client, ActionListener<BulkResponse> listener) {
this.retryOnThrowable = retryOnThrowable;
this.backoff = backoffPolicy.iterator();
this.client = client;
this.listener = listener;
this.logger = Loggers.getLogger(getClass(), client.settings());
// in contrast to System.currentTimeMillis(), nanoTime() uses a monotonic clock under the hood
this.startTimestampNanos = System.nanoTime();
}
@Override
public void onResponse(BulkResponse bulkItemResponses) {
if (!bulkItemResponses.hasFailures()) {
// we're done here, include all responses
addResponses(bulkItemResponses, (r -> true));
finishHim();
} else {
if (canRetry(bulkItemResponses)) {
addResponses(bulkItemResponses, (r -> !r.isFailed()));
retry(createBulkRequestForRetry(bulkItemResponses));
} else {
addResponses(bulkItemResponses, (r -> true));
finishHim();
}
}
}
@Override
public void onFailure(Throwable e) {
try {
listener.onFailure(e);
} finally {
FutureUtils.cancel(scheduledRequestFuture);
}
}
private void retry(BulkRequest bulkRequestForRetry) {
assert backoff.hasNext();
TimeValue next = backoff.next();
logger.trace("Retry of bulk request scheduled in {} ms.", next.millis());
scheduledRequestFuture = client.threadPool().schedule(next, ThreadPool.Names.SAME, (() -> this.execute(bulkRequestForRetry)));
}
private BulkRequest createBulkRequestForRetry(BulkResponse bulkItemResponses) {
BulkRequest requestToReissue = new BulkRequest();
int index = 0;
for (BulkItemResponse bulkItemResponse : bulkItemResponses.getItems()) {
if (bulkItemResponse.isFailed()) {
requestToReissue.add(currentBulkRequest.requests().get(index));
}
index++;
}
return requestToReissue;
}
private boolean canRetry(BulkResponse bulkItemResponses) {
if (!backoff.hasNext()) {
return false;
}
for (BulkItemResponse bulkItemResponse : bulkItemResponses) {
if (bulkItemResponse.isFailed()) {
Throwable cause = bulkItemResponse.getFailure().getCause();
Throwable rootCause = ExceptionsHelper.unwrapCause(cause);
if (!rootCause.getClass().equals(retryOnThrowable)) {
return false;
}
}
}
return true;
}
private void finishHim() {
try {
listener.onResponse(getAccumulatedResponse());
} finally {
FutureUtils.cancel(scheduledRequestFuture);
}
}
private void addResponses(BulkResponse response, Predicate<BulkItemResponse> filter) {
for (BulkItemResponse bulkItemResponse : response) {
if (filter.test(bulkItemResponse)) {
// Use client-side lock here to avoid visibility issues. This method may be called multiple times
// (based on how many retries we have to issue) and relying that the response handling code will be
// scheduled on the same thread is fragile.
synchronized (responses) {
responses.add(bulkItemResponse);
}
}
}
}
private BulkResponse getAccumulatedResponse() {
BulkItemResponse[] itemResponses;
synchronized (responses) {
itemResponses = responses.toArray(new BulkItemResponse[1]);
}
long stopTimestamp = System.nanoTime();
long totalLatencyMs = TimeValue.timeValueNanos(stopTimestamp - startTimestampNanos).millis();
return new BulkResponse(itemResponses, totalLatencyMs);
}
public void execute(BulkRequest bulkRequest) {
this.currentBulkRequest = bulkRequest;
client.bulk(bulkRequest, this);
}
}
static class AsyncRetryHandler extends AbstractRetryHandler {
public AsyncRetryHandler(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy, Client client, ActionListener<BulkResponse> listener) {
super(retryOnThrowable, backoffPolicy, client, listener);
}
}
static class SyncRetryHandler extends AbstractRetryHandler {
private final PlainActionFuture<BulkResponse> actionFuture;
public static SyncRetryHandler create(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy, Client client) {
PlainActionFuture<BulkResponse> actionFuture = PlainActionFuture.newFuture();
return new SyncRetryHandler(retryOnThrowable, backoffPolicy, client, actionFuture);
}
public SyncRetryHandler(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy, Client client, PlainActionFuture<BulkResponse> actionFuture) {
super(retryOnThrowable, backoffPolicy, client, actionFuture);
this.actionFuture = actionFuture;
}
public ActionFuture<BulkResponse> executeBlocking(BulkRequest bulkRequest) {
super.execute(bulkRequest);
return actionFuture;
}
}
}

3
core/src/main/java/org/elasticsearch/action/percolate/TransportShardMultiPercolateAction.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.action.percolate;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.IndicesRequest;
@ -37,8 +36,6 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.text.StringText;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.percolator.PercolatorService;
import org.elasticsearch.threadpool.ThreadPool;

8
core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
View File

@ -473,6 +473,14 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
return this;
}
/**
* Should the query be profiled. Defaults to <code>false</code>
*/
public SearchRequestBuilder setProfile(boolean profile) {
sourceBuilder().profile(profile);
return this;
}
@Override
public String toString() {
if (request.source() != null) {

14
core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.action.search;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.unit.TimeValue;
@ -32,9 +33,12 @@ import org.elasticsearch.rest.action.support.RestActions;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.internal.InternalSearchResponse;
import org.elasticsearch.search.profile.ProfileShardResult;
import org.elasticsearch.search.suggest.Suggest;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.action.search.ShardSearchFailure.readShardSearchFailure;
import static org.elasticsearch.search.internal.InternalSearchResponse.readInternalSearchResponse;
@ -160,6 +164,16 @@ public class SearchResponse extends ActionResponse implements StatusToXContent {
this.scrollId = scrollId;
}
/**
* If profiling was enabled, this returns an object containing the profile results from
* each shard. If profiling was not enabled, this will return null
*
* @return The profile results or null
*/
public @Nullable Map<String, List<ProfileShardResult>> getProfileResults() {
return internalResponse.profile();
}
static final class Fields {
static final XContentBuilderString _SCROLL_ID = new XContentBuilderString("_scroll_id");
static final XContentBuilderString TOOK = new XContentBuilderString("took");

26
core/src/main/java/org/elasticsearch/action/support/DestructiveOperations.java
View File

@ -21,25 +21,30 @@ package org.elasticsearch.action.support;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.settings.NodeSettingsService;
/**
* Helper for dealing with destructive operations and wildcard usage.
*/
public final class DestructiveOperations extends AbstractComponent implements NodeSettingsService.Listener {
public final class DestructiveOperations extends AbstractComponent {
/**
* Setting which controls whether wildcard usage (*, prefix*, _all) is allowed.
*/
public static final String REQUIRES_NAME = "action.destructive_requires_name";
public static final Setting<Boolean> REQUIRES_NAME_SETTING = Setting.boolSetting("action.destructive_requires_name", false, true, Setting.Scope.CLUSTER);
private volatile boolean destructiveRequiresName;
@Inject
public DestructiveOperations(Settings settings, NodeSettingsService nodeSettingsService) {
public DestructiveOperations(Settings settings, ClusterSettings clusterSettings) {
super(settings);
destructiveRequiresName = settings.getAsBoolean(DestructiveOperations.REQUIRES_NAME, false);
nodeSettingsService.addListener(this);
destructiveRequiresName = REQUIRES_NAME_SETTING.get(settings);
clusterSettings.addSettingsUpdateConsumer(REQUIRES_NAME_SETTING, this::setDestructiveRequiresName);
}
private void setDestructiveRequiresName(boolean destructiveRequiresName) {
this.destructiveRequiresName = destructiveRequiresName;
}
/**
@ -65,15 +70,6 @@ public final class DestructiveOperations extends AbstractComponent implements No
}
}
@Override
public void onRefreshSettings(Settings settings) {
boolean newValue = settings.getAsBoolean(DestructiveOperations.REQUIRES_NAME, destructiveRequiresName);
if (destructiveRequiresName != newValue) {
logger.info("updating [action.operate_all_indices] from [{}] to [{}]", destructiveRequiresName, newValue);
this.destructiveRequiresName = newValue;
}
}
private static boolean hasWildcardUsage(String aliasOrIndex) {
return "_all".equals(aliasOrIndex) || aliasOrIndex.indexOf('*') != -1;
}

31
core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
View File

@ -19,8 +19,16 @@
package org.elasticsearch.action.support.broadcast.node;
import org.elasticsearch.action.*;
import org.elasticsearch.action.support.*;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.FailedNodeException;
import org.elasticsearch.action.IndicesRequest;
import org.elasticsearch.action.NoShardAvailableActionException;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.DefaultShardOperationFailedException;
import org.elasticsearch.action.support.HandledTransportAction;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.TransportActions;
import org.elasticsearch.action.support.broadcast.BroadcastRequest;
import org.elasticsearch.action.support.broadcast.BroadcastResponse;
import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
@ -37,7 +45,14 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.*;
import org.elasticsearch.transport.BaseTransportResponseHandler;
import org.elasticsearch.transport.NodeShouldNotConnectException;
import org.elasticsearch.transport.TransportChannel;
import org.elasticsearch.transport.TransportException;
import org.elasticsearch.transport.TransportRequest;
import org.elasticsearch.transport.TransportRequestHandler;
import org.elasticsearch.transport.TransportResponse;
import org.elasticsearch.transport.TransportService;
import java.io.IOException;
import java.util.ArrayList;
@ -394,7 +409,15 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
e.setIndex(shardRouting.getIndex());
e.setShard(shardRouting.shardId());
shardResults[shardIndex] = e;
logger.debug("[{}] failed to execute operation for shard [{}]", e, actionName, shardRouting.shortSummary());
if (TransportActions.isShardNotAvailableException(t)) {
if (logger.isTraceEnabled()) {
logger.trace("[{}] failed to execute operation for shard [{}]", t, actionName, shardRouting.shortSummary());
}
} else {
if (logger.isDebugEnabled()) {
logger.debug("[{}] failed to execute operation for shard [{}]", t, actionName, shardRouting.shortSummary());
}
}
}
}
}

8
core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
View File

@ -300,11 +300,15 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
@Override
public void onFailure(Throwable t) {
if (t instanceof RetryOnReplicaException) {
logger.trace("Retrying operation on replica, action [{}], request [{}]", t, actionName, request);
logger.trace("Retrying operation on replica, action [{}], request [{}]", t, transportReplicaAction, request);
observer.waitForNextChange(new ClusterStateObserver.Listener() {
@Override
public void onNewClusterState(ClusterState state) {
threadPool.executor(executor).execute(AsyncReplicaAction.this);
// Forking a thread on local node via transport service so that custom transport service have an
// opportunity to execute custom logic before the replica operation begins
String extraMessage = "action [" + transportReplicaAction + "], request[" + request + "]";
TransportChannelResponseHandler<TransportResponse.Empty> handler = TransportChannelResponseHandler.emptyResponseHandler(logger, channel, extraMessage);
transportService.sendRequest(clusterService.localNode(), transportReplicaAction, request, handler);
}
@Override

3
core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
View File

@ -50,6 +50,7 @@ import org.elasticsearch.search.fetch.source.FetchSourceContext;
import org.elasticsearch.search.lookup.SourceLookup;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@ -245,7 +246,7 @@ public class UpdateHelper extends AbstractComponent {
private Map<String, Object> executeScript(UpdateRequest request, Map<String, Object> ctx) {
try {
if (scriptService != null) {
ExecutableScript script = scriptService.executable(request.script, ScriptContext.Standard.UPDATE, request);
ExecutableScript script = scriptService.executable(request.script, ScriptContext.Standard.UPDATE, request, Collections.emptyMap());
script.setNextVar("ctx", ctx);
script.run();
// we need to unwrap the ctx...

30
core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
View File

@ -53,8 +53,8 @@ import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse;
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder;
import org.elasticsearch.action.admin.indices.flush.FlushResponse;
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder;
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder;
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder;
@ -82,11 +82,14 @@ import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsResponse;
import org.elasticsearch.action.admin.indices.shards.IndicesShardStoreRequestBuilder;
import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresRequest;
import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest;
import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequestBuilder;
import org.elasticsearch.action.admin.indices.flush.SyncedFlushResponse;
import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder;
import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse;
@ -390,6 +393,29 @@ public interface IndicesAdminClient extends ElasticsearchClient {
*/
FlushRequestBuilder prepareFlush(String... indices);
/**
* Explicitly sync flush one or more indices (write sync id to shards for faster recovery).
*
* @param request The sync flush request
* @return A result future
* @see org.elasticsearch.client.Requests#syncedFlushRequest(String...)
*/
ActionFuture<SyncedFlushResponse> syncedFlush(SyncedFlushRequest request);
/**
* Explicitly sync flush one or more indices (write sync id to shards for faster recovery).
*
* @param request The sync flush request
* @param listener A listener to be notified with a result
* @see org.elasticsearch.client.Requests#syncedFlushRequest(String...)
*/
void syncedFlush(SyncedFlushRequest request, ActionListener <SyncedFlushResponse> listener);
/**
* Explicitly sync flush one or more indices (write sync id to shards for faster recovery).
*/
SyncedFlushRequestBuilder prepareSyncedFlush(String... indices);
/**
* Explicitly force merge one or more indices into a the number of segments.
*

14
core/src/main/java/org/elasticsearch/client/Requests.java
View File

@ -50,6 +50,7 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresRequest;
import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest;
import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.delete.DeleteRequest;
@ -131,7 +132,7 @@ public class Requests {
public static SuggestRequest suggestRequest(String... indices) {
return new SuggestRequest(indices);
}
/**
* Creates a search request against one or more indices. Note, the search source must be set either using the
* actual JSON search source, or the {@link org.elasticsearch.search.builder.SearchSourceBuilder}.
@ -265,6 +266,17 @@ public class Requests {
return new FlushRequest(indices);
}
/**
* Creates a synced flush indices request.
*
* @param indices The indices to sync flush. Use <tt>null</tt> or <tt>_all</tt> to execute against all indices
* @return The synced flush request
* @see org.elasticsearch.client.IndicesAdminClient#syncedFlush(SyncedFlushRequest)
*/
public static SyncedFlushRequest syncedFlushRequest(String... indices) {
return new SyncedFlushRequest(indices);
}
/**
* Creates a force merge request.
*

19
core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
View File

@ -188,6 +188,10 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.admin.indices.flush.SyncedFlushAction;
import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest;
import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequestBuilder;
import org.elasticsearch.action.admin.indices.flush.SyncedFlushResponse;
import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateAction;
import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder;
@ -1315,6 +1319,21 @@ public abstract class AbstractClient extends AbstractComponent implements Client
return new FlushRequestBuilder(this, FlushAction.INSTANCE).setIndices(indices);
}
@Override
public ActionFuture<SyncedFlushResponse> syncedFlush(SyncedFlushRequest request) {
return execute(SyncedFlushAction.INSTANCE, request);
}
@Override
public void syncedFlush(SyncedFlushRequest request, ActionListener<SyncedFlushResponse> listener) {
execute(SyncedFlushAction.INSTANCE, request, listener);
}
@Override
public SyncedFlushRequestBuilder prepareSyncedFlush(String... indices) {
return new SyncedFlushRequestBuilder(this, SyncedFlushAction.INSTANCE).setIndices(indices);
}
@Override
public void getMappings(GetMappingsRequest request, ActionListener<GetMappingsResponse> listener) {
execute(GetMappingsAction.INSTANCE, request, listener);

11
core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
View File

@ -32,7 +32,6 @@ import org.elasticsearch.client.support.Headers;
import org.elasticsearch.client.transport.support.TransportProxyClient;
import org.elasticsearch.cluster.ClusterNameModule;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.common.inject.Module;
@ -43,19 +42,15 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsFilter;
import org.elasticsearch.common.settings.SettingsModule;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.EnvironmentModule;
import org.elasticsearch.indices.breaker.CircuitBreakerModule;
import org.elasticsearch.monitor.MonitorService;
import org.elasticsearch.node.internal.InternalSettingsPreparer;
import org.elasticsearch.node.settings.NodeSettingsService;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.PluginsModule;
import org.elasticsearch.plugins.PluginsService;
import org.elasticsearch.search.SearchModule;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.threadpool.ThreadPoolModule;
import org.elasticsearch.transport.TransportModule;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.transport.netty.NettyTransport;
@ -69,7 +64,7 @@ import static org.elasticsearch.common.settings.Settings.settingsBuilder;
* The transport client allows to create a client that is not part of the cluster, but simply connects to one
* or more nodes directly by adding their respective addresses using {@link #addTransportAddress(org.elasticsearch.common.transport.TransportAddress)}.
* <p>
* The transport client important modules used is the {@link org.elasticsearch.transport.TransportModule} which is
* The transport client important modules used is the {@link org.elasticsearch.common.network.NetworkModule} which is
* started in client mode (only connects, no bind).
*/
public class TransportClient extends AbstractClient {
@ -143,10 +138,9 @@ public class TransportClient extends AbstractClient {
}
modules.add(new PluginsModule(pluginsService));
modules.add(new SettingsModule(this.settings, settingsFilter ));
modules.add(new NetworkModule(networkService));
modules.add(new NetworkModule(networkService, this.settings, true));
modules.add(new ClusterNameModule(this.settings));
modules.add(new ThreadPoolModule(threadPool));
modules.add(new TransportModule(this.settings));
modules.add(new SearchModule() {
@Override
protected void configure() {
@ -154,7 +148,6 @@ public class TransportClient extends AbstractClient {
}
});
modules.add(new ActionModule(true));
modules.add(new ClientTransportModule());
modules.add(new CircuitBreakerModule(this.settings));
pluginsService.processModules(modules);

87
core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
View File

@ -19,9 +19,6 @@
package org.elasticsearch.cluster;
import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction;
import org.elasticsearch.action.support.DestructiveOperations;
import org.elasticsearch.action.support.replication.TransportReplicationAction;
import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction;
import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
@ -29,7 +26,6 @@ import org.elasticsearch.cluster.action.shard.ShardStateAction;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.IndexTemplateFilter;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
import org.elasticsearch.cluster.metadata.MetaDataDeleteIndexService;
import org.elasticsearch.cluster.metadata.MetaDataIndexAliasesService;
@ -60,17 +56,15 @@ import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocatio
import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider;
import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
import org.elasticsearch.cluster.service.InternalClusterService;
import org.elasticsearch.cluster.settings.ClusterDynamicSettings;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.cluster.settings.DynamicSettings;
import org.elasticsearch.cluster.settings.Validator;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.ExtensionPoint;
import org.elasticsearch.discovery.DiscoverySettings;
import org.elasticsearch.discovery.zen.ZenDiscovery;
import org.elasticsearch.discovery.zen.elect.ElectMasterService;
import org.elasticsearch.gateway.GatewayAllocator;
import org.elasticsearch.gateway.PrimaryShardAllocator;
import org.elasticsearch.index.engine.EngineConfig;
@ -81,21 +75,13 @@ import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.MergePolicyConfig;
import org.elasticsearch.index.shard.MergeSchedulerConfig;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.index.store.IndexStoreConfig;
import org.elasticsearch.index.translog.TranslogConfig;
import org.elasticsearch.indices.IndicesWarmer;
import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
import org.elasticsearch.indices.cache.request.IndicesRequestCache;
import org.elasticsearch.indices.recovery.RecoverySettings;
import org.elasticsearch.indices.ttl.IndicesTTLService;
import org.elasticsearch.search.SearchService;
import org.elasticsearch.search.internal.DefaultSearchContext;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.*;
/**
* Configures classes and services that affect the entire cluster.
@ -122,7 +108,6 @@ public class ClusterModule extends AbstractModule {
SnapshotInProgressAllocationDecider.class));
private final Settings settings;
private final DynamicSettings.Builder clusterDynamicSettings = new DynamicSettings.Builder();
private final DynamicSettings.Builder indexDynamicSettings = new DynamicSettings.Builder();
private final ExtensionPoint.SelectedType<ShardsAllocator> shardsAllocators = new ExtensionPoint.SelectedType<>("shards_allocator", ShardsAllocator.class);
private final ExtensionPoint.ClassSet<AllocationDecider> allocationDeciders = new ExtensionPoint.ClassSet<>("allocation_decider", AllocationDecider.class, AllocationDeciders.class);
@ -134,7 +119,6 @@ public class ClusterModule extends AbstractModule {
public ClusterModule(Settings settings) {
this.settings = settings;
registerBuiltinClusterSettings();
registerBuiltinIndexSettings();
for (Class<? extends AllocationDecider> decider : ClusterModule.DEFAULT_ALLOCATION_DECIDERS) {
@ -144,70 +128,11 @@ public class ClusterModule extends AbstractModule {
registerShardsAllocator(ClusterModule.EVEN_SHARD_COUNT_ALLOCATOR, BalancedShardsAllocator.class);
}
private void registerBuiltinClusterSettings() {
registerClusterDynamicSetting(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTES, Validator.EMPTY);
registerClusterDynamicSetting(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP + "*", Validator.EMPTY);
registerClusterDynamicSetting(BalancedShardsAllocator.SETTING_INDEX_BALANCE_FACTOR, Validator.FLOAT);
registerClusterDynamicSetting(BalancedShardsAllocator.SETTING_SHARD_BALANCE_FACTOR, Validator.FLOAT);
registerClusterDynamicSetting(BalancedShardsAllocator.SETTING_THRESHOLD, Validator.NON_NEGATIVE_FLOAT);
registerClusterDynamicSetting(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, ClusterRebalanceAllocationDecider.ALLOCATION_ALLOW_REBALANCE_VALIDATOR);
registerClusterDynamicSetting(ConcurrentRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE, Validator.INTEGER);
registerClusterDynamicSetting(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, Validator.EMPTY);
registerClusterDynamicSetting(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE, Validator.EMPTY);
registerClusterDynamicSetting(ZenDiscovery.SETTING_REJOIN_ON_MASTER_GONE, Validator.BOOLEAN);
registerClusterDynamicSetting(DiscoverySettings.NO_MASTER_BLOCK, Validator.EMPTY);
registerClusterDynamicSetting(FilterAllocationDecider.CLUSTER_ROUTING_INCLUDE_GROUP + "*", Validator.EMPTY);
registerClusterDynamicSetting(FilterAllocationDecider.CLUSTER_ROUTING_EXCLUDE_GROUP + "*", Validator.EMPTY);
registerClusterDynamicSetting(FilterAllocationDecider.CLUSTER_ROUTING_REQUIRE_GROUP + "*", Validator.EMPTY);
registerClusterDynamicSetting(IndexStoreConfig.INDICES_STORE_THROTTLE_TYPE, Validator.EMPTY);
registerClusterDynamicSetting(IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, Validator.BYTES_SIZE);
registerClusterDynamicSetting(IndicesTTLService.INDICES_TTL_INTERVAL, Validator.TIME);
registerClusterDynamicSetting(MappingUpdatedAction.INDICES_MAPPING_DYNAMIC_TIMEOUT, Validator.TIME);
registerClusterDynamicSetting(MetaData.SETTING_READ_ONLY, Validator.EMPTY);
registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_CONCURRENT_STREAMS, Validator.POSITIVE_INTEGER);
registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_CONCURRENT_SMALL_FILE_STREAMS, Validator.POSITIVE_INTEGER);
registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC, Validator.BYTES_SIZE);
registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC, Validator.TIME_NON_NEGATIVE);
registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK, Validator.TIME_NON_NEGATIVE);
registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_ACTIVITY_TIMEOUT, Validator.TIME_NON_NEGATIVE);
registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT, Validator.TIME_NON_NEGATIVE);
registerClusterDynamicSetting(RecoverySettings.INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT, Validator.TIME_NON_NEGATIVE);
registerClusterDynamicSetting(ThreadPool.THREADPOOL_GROUP + "*", ThreadPool.THREAD_POOL_TYPE_SETTINGS_VALIDATOR);
registerClusterDynamicSetting(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES, Validator.INTEGER);
registerClusterDynamicSetting(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES, Validator.INTEGER);
registerClusterDynamicSetting(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK, Validator.EMPTY);
registerClusterDynamicSetting(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK, Validator.EMPTY);
registerClusterDynamicSetting(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED, Validator.BOOLEAN);
registerClusterDynamicSetting(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS, Validator.BOOLEAN);
registerClusterDynamicSetting(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL, Validator.TIME_NON_NEGATIVE);
registerClusterDynamicSetting(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL, Validator.TIME_NON_NEGATIVE);
registerClusterDynamicSetting(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT, Validator.TIME_NON_NEGATIVE);
registerClusterDynamicSetting(SnapshotInProgressAllocationDecider.CLUSTER_ROUTING_ALLOCATION_SNAPSHOT_RELOCATION_ENABLED, Validator.EMPTY);
registerClusterDynamicSetting(DestructiveOperations.REQUIRES_NAME, Validator.EMPTY);
registerClusterDynamicSetting(DiscoverySettings.PUBLISH_TIMEOUT, Validator.TIME_NON_NEGATIVE);
registerClusterDynamicSetting(DiscoverySettings.PUBLISH_DIFF_ENABLE, Validator.BOOLEAN);
registerClusterDynamicSetting(DiscoverySettings.COMMIT_TIMEOUT, Validator.TIME_NON_NEGATIVE);
registerClusterDynamicSetting(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING, Validator.MEMORY_SIZE);
registerClusterDynamicSetting(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING, Validator.MEMORY_SIZE);
registerClusterDynamicSetting(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING, Validator.NON_NEGATIVE_DOUBLE);
registerClusterDynamicSetting(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING, Validator.MEMORY_SIZE);
registerClusterDynamicSetting(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING, Validator.NON_NEGATIVE_DOUBLE);
registerClusterDynamicSetting(InternalClusterService.SETTING_CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD, Validator.TIME_NON_NEGATIVE);
registerClusterDynamicSetting(SearchService.DEFAULT_SEARCH_TIMEOUT, Validator.TIMEOUT);
registerClusterDynamicSetting(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES, ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_VALIDATOR);
registerClusterDynamicSetting(TransportService.SETTING_TRACE_LOG_INCLUDE, Validator.EMPTY);
registerClusterDynamicSetting(TransportService.SETTING_TRACE_LOG_INCLUDE + ".*", Validator.EMPTY);
registerClusterDynamicSetting(TransportService.SETTING_TRACE_LOG_EXCLUDE, Validator.EMPTY);
registerClusterDynamicSetting(TransportService.SETTING_TRACE_LOG_EXCLUDE + ".*", Validator.EMPTY);
registerClusterDynamicSetting(TransportCloseIndexAction.SETTING_CLUSTER_INDICES_CLOSE_ENABLE, Validator.BOOLEAN);
registerClusterDynamicSetting(ShardsLimitAllocationDecider.CLUSTER_TOTAL_SHARDS_PER_NODE, Validator.INTEGER);
registerClusterDynamicSetting(TransportReplicationAction.SHARD_FAILURE_TIMEOUT, Validator.TIME_NON_NEGATIVE);
}
private void registerBuiltinIndexSettings() {
registerIndexDynamicSetting(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, Validator.BYTES_SIZE);
registerIndexDynamicSetting(IndexStore.INDEX_STORE_THROTTLE_TYPE, Validator.EMPTY);
registerIndexDynamicSetting(MergeSchedulerConfig.MAX_THREAD_COUNT, Validator.EMPTY);
registerIndexDynamicSetting(MergeSchedulerConfig.MAX_THREAD_COUNT, Validator.NON_NEGATIVE_INTEGER);
registerIndexDynamicSetting(MergeSchedulerConfig.MAX_MERGE_COUNT, Validator.EMPTY);
registerIndexDynamicSetting(MergeSchedulerConfig.AUTO_THROTTLE, Validator.EMPTY);
registerIndexDynamicSetting(FilterAllocationDecider.INDEX_ROUTING_REQUIRE_GROUP + "*", Validator.EMPTY);
@ -272,9 +197,6 @@ public class ClusterModule extends AbstractModule {
indexDynamicSettings.addSetting(setting, validator);
}
public void registerClusterDynamicSetting(String setting, Validator validator) {
clusterDynamicSettings.addSetting(setting, validator);
}
public void registerAllocationDecider(Class<? extends AllocationDecider> allocationDecider) {
allocationDeciders.registerExtension(allocationDecider);
@ -290,7 +212,6 @@ public class ClusterModule extends AbstractModule {
@Override
protected void configure() {
bind(DynamicSettings.class).annotatedWith(ClusterDynamicSettings.class).toInstance(clusterDynamicSettings.build());
bind(DynamicSettings.class).annotatedWith(IndexDynamicSettings.class).toInstance(indexDynamicSettings.build());
// bind ShardsAllocator

2
core/src/main/java/org/elasticsearch/cluster/ClusterState.java
View File

@ -129,7 +129,7 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
@SuppressWarnings("unchecked")
T proto = (T)customPrototypes.get(type);
if (proto == null) {
throw new IllegalArgumentException("No custom state prototype registered for type [" + type + "]");
throw new IllegalArgumentException("No custom state prototype registered for type [" + type + "], node likely missing plugins");
}
return proto;
}

7
core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
View File

@ -37,6 +37,13 @@ public interface ClusterStateTaskExecutor<T> {
return true;
}
/**
* Callback invoked after new cluster state is published. Note that
* this method is not invoked if the cluster state was not updated.
*/
default void clusterStatePublished(ClusterState newClusterState) {
}
/**
* Represents the result of a batched execution of cluster state update tasks
* @param <T> the type of the cluster state update task

54
core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
View File

@ -37,11 +37,12 @@ import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
import org.elasticsearch.monitor.fs.FsInfo;
import org.elasticsearch.node.settings.NodeSettingsService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.ReceiveTimeoutTransportException;
@ -63,8 +64,8 @@ import java.util.concurrent.TimeUnit;
*/
public class InternalClusterInfoService extends AbstractComponent implements ClusterInfoService, LocalNodeMasterListener, ClusterStateListener {
public static final String INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL = "cluster.info.update.interval";
public static final String INTERNAL_CLUSTER_INFO_TIMEOUT = "cluster.info.update.timeout";
public static final Setting<TimeValue> INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING = Setting.timeSetting("cluster.info.update.interval", TimeValue.timeValueSeconds(30), TimeValue.timeValueSeconds(10), true, Setting.Scope.CLUSTER);
public static final Setting<TimeValue> INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING = Setting.positiveTimeSetting("cluster.info.update.timeout", TimeValue.timeValueSeconds(15), true, Setting.Scope.CLUSTER);
private volatile TimeValue updateFrequency;
@ -82,7 +83,7 @@ public class InternalClusterInfoService extends AbstractComponent implements Clu
private final List<Listener> listeners = new CopyOnWriteArrayList<>();
@Inject
public InternalClusterInfoService(Settings settings, NodeSettingsService nodeSettingsService,
public InternalClusterInfoService(Settings settings, ClusterSettings clusterSettings,
TransportNodesStatsAction transportNodesStatsAction,
TransportIndicesStatsAction transportIndicesStatsAction, ClusterService clusterService,
ThreadPool threadPool) {
@ -95,10 +96,12 @@ public class InternalClusterInfoService extends AbstractComponent implements Clu
this.transportIndicesStatsAction = transportIndicesStatsAction;
this.clusterService = clusterService;
this.threadPool = threadPool;
this.updateFrequency = settings.getAsTime(INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL, TimeValue.timeValueSeconds(30));
this.fetchTimeout = settings.getAsTime(INTERNAL_CLUSTER_INFO_TIMEOUT, TimeValue.timeValueSeconds(15));
this.enabled = settings.getAsBoolean(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED, true);
nodeSettingsService.addListener(new ApplySettings());
this.updateFrequency = INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.get(settings);
this.fetchTimeout = INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.get(settings);
this.enabled = DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.get(settings);
clusterSettings.addSettingsUpdateConsumer(INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING, this::setFetchTimeout);
clusterSettings.addSettingsUpdateConsumer(INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING, this::setUpdateFrequency);
clusterSettings.addSettingsUpdateConsumer(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING, this::setEnabled);
// Add InternalClusterInfoService to listen for Master changes
this.clusterService.add((LocalNodeMasterListener)this);
@ -106,35 +109,16 @@ public class InternalClusterInfoService extends AbstractComponent implements Clu
this.clusterService.add((ClusterStateListener)this);
}
class ApplySettings implements NodeSettingsService.Listener {
@Override
public void onRefreshSettings(Settings settings) {
TimeValue newUpdateFrequency = settings.getAsTime(INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL, null);
// ClusterInfoService is only enabled if the DiskThresholdDecider is enabled
Boolean newEnabled = settings.getAsBoolean(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED, null);
private void setEnabled(boolean enabled) {
this.enabled = enabled;
}
if (newUpdateFrequency != null) {
if (newUpdateFrequency.getMillis() < TimeValue.timeValueSeconds(10).getMillis()) {
logger.warn("[{}] set too low [{}] (< 10s)", INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL, newUpdateFrequency);
throw new IllegalStateException("Unable to set " + INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL + " less than 10 seconds");
} else {
logger.info("updating [{}] from [{}] to [{}]", INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL, updateFrequency, newUpdateFrequency);
InternalClusterInfoService.this.updateFrequency = newUpdateFrequency;
}
}
private void setFetchTimeout(TimeValue fetchTimeout) {
this.fetchTimeout = fetchTimeout;
}
TimeValue newFetchTimeout = settings.getAsTime(INTERNAL_CLUSTER_INFO_TIMEOUT, null);
if (newFetchTimeout != null) {
logger.info("updating fetch timeout [{}] from [{}] to [{}]", INTERNAL_CLUSTER_INFO_TIMEOUT, fetchTimeout, newFetchTimeout);
InternalClusterInfoService.this.fetchTimeout = newFetchTimeout;
}
// We don't log about enabling it here, because the DiskThresholdDecider will already be logging about enable/disable
if (newEnabled != null) {
InternalClusterInfoService.this.enabled = newEnabled;
}
}
void setUpdateFrequency(TimeValue updateFrequency) {
this.updateFrequency = updateFrequency;
}
@Override

28
core/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java
View File

@ -26,11 +26,12 @@ import org.elasticsearch.client.Client;
import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.Mapping;
import org.elasticsearch.node.settings.NodeSettingsService;
import java.util.concurrent.TimeoutException;
@ -40,30 +41,23 @@ import java.util.concurrent.TimeoutException;
*/
public class MappingUpdatedAction extends AbstractComponent {
public static final String INDICES_MAPPING_DYNAMIC_TIMEOUT = "indices.mapping.dynamic_timeout";
public static final Setting<TimeValue> INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING = Setting.positiveTimeSetting("indices.mapping.dynamic_timeout", TimeValue.timeValueSeconds(30), true, Setting.Scope.CLUSTER);
private IndicesAdminClient client;
private volatile TimeValue dynamicMappingUpdateTimeout;
class ApplySettings implements NodeSettingsService.Listener {
@Override
public void onRefreshSettings(Settings settings) {
TimeValue current = MappingUpdatedAction.this.dynamicMappingUpdateTimeout;
TimeValue newValue = settings.getAsTime(INDICES_MAPPING_DYNAMIC_TIMEOUT, current);
if (!current.equals(newValue)) {
logger.info("updating " + INDICES_MAPPING_DYNAMIC_TIMEOUT + " from [{}] to [{}]", current, newValue);
MappingUpdatedAction.this.dynamicMappingUpdateTimeout = newValue;
}
}
@Inject
public MappingUpdatedAction(Settings settings, ClusterSettings clusterSettings) {
super(settings);
this.dynamicMappingUpdateTimeout = INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING.get(settings);
clusterSettings.addSettingsUpdateConsumer(INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING, this::setDynamicMappingUpdateTimeout);
}
@Inject
public MappingUpdatedAction(Settings settings, NodeSettingsService nodeSettingsService) {
super(settings);
this.dynamicMappingUpdateTimeout = settings.getAsTime(INDICES_MAPPING_DYNAMIC_TIMEOUT, TimeValue.timeValueSeconds(30));
nodeSettingsService.addListener(new ApplySettings());
private void setDynamicMappingUpdateTimeout(TimeValue dynamicMappingUpdateTimeout) {
this.dynamicMappingUpdateTimeout = dynamicMappingUpdateTimeout;
}
public void setClient(Client client) {
this.client = client.admin().indices();
}

230
core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
View File

@ -20,7 +20,12 @@
package org.elasticsearch.cluster.action.shard;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.cluster.*;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateTaskConfig;
import org.elasticsearch.cluster.ClusterStateTaskExecutor;
import org.elasticsearch.cluster.ClusterStateTaskListener;
import org.elasticsearch.cluster.NotMasterException;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.RoutingService;
@ -37,19 +42,24 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.*;
import org.elasticsearch.transport.EmptyTransportResponseHandler;
import org.elasticsearch.transport.TransportChannel;
import org.elasticsearch.transport.TransportException;
import org.elasticsearch.transport.TransportRequest;
import org.elasticsearch.transport.TransportRequestHandler;
import org.elasticsearch.transport.TransportRequestOptions;
import org.elasticsearch.transport.TransportResponse;
import org.elasticsearch.transport.TransportService;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import static org.elasticsearch.cluster.routing.ShardRouting.readShardRoutingEntry;
/**
*
*/
public class ShardStateAction extends AbstractComponent {
public class ShardStateAction extends AbstractComponent {
public static final String SHARD_STARTED_ACTION_NAME = "internal:cluster/shard/started";
public static final String SHARD_FAILED_ACTION_NAME = "internal:cluster/shard/failure";
@ -97,18 +107,101 @@ public class ShardStateAction extends AbstractComponent {
options = TransportRequestOptions.builder().withTimeout(timeout).build();
}
transportService.sendRequest(masterNode,
SHARD_FAILED_ACTION_NAME, shardRoutingEntry, options, new EmptyTransportResponseHandler(ThreadPool.Names.SAME) {
SHARD_FAILED_ACTION_NAME, shardRoutingEntry, options, new EmptyTransportResponseHandler(ThreadPool.Names.SAME) {
@Override
public void handleResponse(TransportResponse.Empty response) {
listener.onSuccess();
}
@Override
public void handleException(TransportException exp) {
logger.warn("unexpected failure while sending request to [{}] to fail shard [{}]", exp, masterNode, shardRoutingEntry);
listener.onShardFailedFailure(masterNode, exp);
}
});
}
private class ShardFailedTransportHandler implements TransportRequestHandler<ShardRoutingEntry> {
@Override
public void messageReceived(ShardRoutingEntry request, TransportChannel channel) throws Exception {
handleShardFailureOnMaster(request, new ClusterStateTaskListener() {
@Override
public void handleResponse(TransportResponse.Empty response) {
listener.onSuccess();
public void onFailure(String source, Throwable t) {
logger.error("unexpected failure while failing shard [{}]", t, request.shardRouting);
try {
channel.sendResponse(t);
} catch (Throwable channelThrowable) {
logger.warn("failed to send failure [{}] while failing shard [{}]", channelThrowable, t, request.shardRouting);
}
}
@Override
public void handleException(TransportException exp) {
logger.warn("failed to send failed shard to {}", exp, masterNode);
listener.onShardFailedFailure(masterNode, exp);
public void onNoLongerMaster(String source) {
logger.error("no longer master while failing shard [{}]", request.shardRouting);
try {
channel.sendResponse(new NotMasterException(source));
} catch (Throwable channelThrowable) {
logger.warn("failed to send no longer master while failing shard [{}]", channelThrowable, request.shardRouting);
}
}
});
@Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
try {
channel.sendResponse(TransportResponse.Empty.INSTANCE);
} catch (Throwable channelThrowable) {
logger.warn("failed to send response while failing shard [{}]", channelThrowable, request.shardRouting);
}
}
}
);
}
}
class ShardFailedClusterStateHandler implements ClusterStateTaskExecutor<ShardRoutingEntry> {
@Override
public BatchResult<ShardRoutingEntry> execute(ClusterState currentState, List<ShardRoutingEntry> tasks) throws Exception {
BatchResult.Builder<ShardRoutingEntry> batchResultBuilder = BatchResult.builder();
List<FailedRerouteAllocation.FailedShard> failedShards = new ArrayList<>(tasks.size());
for (ShardRoutingEntry task : tasks) {
failedShards.add(new FailedRerouteAllocation.FailedShard(task.shardRouting, task.message, task.failure));
}
ClusterState maybeUpdatedState = currentState;
try {
RoutingAllocation.Result result = allocationService.applyFailedShards(currentState, failedShards);
if (result.changed()) {
maybeUpdatedState = ClusterState.builder(currentState).routingResult(result).build();
}
batchResultBuilder.successes(tasks);
} catch (Throwable t) {
batchResultBuilder.failures(tasks, t);
}
return batchResultBuilder.build(maybeUpdatedState);
}
@Override
public void clusterStatePublished(ClusterState newClusterState) {
int numberOfUnassignedShards = newClusterState.getRoutingNodes().unassigned().size();
if (numberOfUnassignedShards > 0) {
String reason = String.format(Locale.ROOT, "[%d] unassigned shards after failing shards", numberOfUnassignedShards);
if (logger.isTraceEnabled()) {
logger.trace(reason + ", scheduling a reroute");
}
routingService.reroute(reason);
}
}
}
private final ShardFailedClusterStateHandler shardFailedClusterStateHandler = new ShardFailedClusterStateHandler();
private void handleShardFailureOnMaster(final ShardRoutingEntry shardRoutingEntry, ClusterStateTaskListener listener) {
logger.warn("{} received shard failed for {}", shardRoutingEntry.failure, shardRoutingEntry.shardRouting.shardId(), shardRoutingEntry);
clusterService.submitStateUpdateTask(
"shard-failed (" + shardRoutingEntry.shardRouting + "), message [" + shardRoutingEntry.message + "]",
shardRoutingEntry,
ClusterStateTaskConfig.build(Priority.HIGH),
shardFailedClusterStateHandler,
listener);
}
public void shardStarted(final ShardRouting shardRouting, String indexUUID, final String reason) {
@ -124,74 +217,20 @@ public class ShardStateAction extends AbstractComponent {
ShardRoutingEntry shardRoutingEntry = new ShardRoutingEntry(shardRouting, indexUUID, reason, null);
logger.debug("{} sending shard started for {}", shardRoutingEntry.shardRouting.shardId(), shardRoutingEntry);
transportService.sendRequest(masterNode,
SHARD_STARTED_ACTION_NAME, new ShardRoutingEntry(shardRouting, indexUUID, reason, null), new EmptyTransportResponseHandler(ThreadPool.Names.SAME) {
@Override
public void handleException(TransportException exp) {
logger.warn("failed to send shard started to [{}]", exp, masterNode);
}
});
}
private final ShardFailedClusterStateHandler shardFailedClusterStateHandler = new ShardFailedClusterStateHandler();
private void handleShardFailureOnMaster(final ShardRoutingEntry shardRoutingEntry) {
logger.warn("{} received shard failed for {}", shardRoutingEntry.failure, shardRoutingEntry.shardRouting.shardId(), shardRoutingEntry);
clusterService.submitStateUpdateTask(
"shard-failed (" + shardRoutingEntry.shardRouting + "), message [" + shardRoutingEntry.message + "]",
shardRoutingEntry,
ClusterStateTaskConfig.build(Priority.HIGH),
shardFailedClusterStateHandler,
shardFailedClusterStateHandler);
}
class ShardFailedClusterStateHandler implements ClusterStateTaskExecutor<ShardRoutingEntry>, ClusterStateTaskListener {
@Override
public BatchResult<ShardRoutingEntry> execute(ClusterState currentState, List<ShardRoutingEntry> tasks) throws Exception {
BatchResult.Builder<ShardRoutingEntry> batchResultBuilder = BatchResult.builder();
List<FailedRerouteAllocation.FailedShard> shardRoutingsToBeApplied = new ArrayList<>(tasks.size());
for (ShardRoutingEntry task : tasks) {
shardRoutingsToBeApplied.add(new FailedRerouteAllocation.FailedShard(task.shardRouting, task.message, task.failure));
}
ClusterState maybeUpdatedState = currentState;
try {
RoutingAllocation.Result result = allocationService.applyFailedShards(currentState, shardRoutingsToBeApplied);
if (result.changed()) {
maybeUpdatedState = ClusterState.builder(currentState).routingResult(result).build();
SHARD_STARTED_ACTION_NAME, new ShardRoutingEntry(shardRouting, indexUUID, reason, null), new EmptyTransportResponseHandler(ThreadPool.Names.SAME) {
@Override
public void handleException(TransportException exp) {
logger.warn("failed to send shard started to [{}]", exp, masterNode);
}
batchResultBuilder.successes(tasks);
} catch (Throwable t) {
batchResultBuilder.failures(tasks, t);
}
return batchResultBuilder.build(maybeUpdatedState);
}
@Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
if (oldState != newState && newState.getRoutingNodes().unassigned().size() > 0) {
logger.trace("unassigned shards after shard failures. scheduling a reroute.");
routingService.reroute("unassigned shards after shard failures, scheduling a reroute");
}
}
@Override
public void onFailure(String source, Throwable t) {
logger.error("unexpected failure during [{}]", t, source);
}
});
}
private final ShardStartedClusterStateHandler shardStartedClusterStateHandler =
new ShardStartedClusterStateHandler();
private void shardStartedOnMaster(final ShardRoutingEntry shardRoutingEntry) {
logger.debug("received shard started for {}", shardRoutingEntry);
clusterService.submitStateUpdateTask(
"shard-started (" + shardRoutingEntry.shardRouting + "), reason [" + shardRoutingEntry.message + "]",
shardRoutingEntry,
ClusterStateTaskConfig.build(Priority.URGENT),
shardStartedClusterStateHandler,
shardStartedClusterStateHandler);
class ShardStartedTransportHandler implements TransportRequestHandler<ShardRoutingEntry> {
@Override
public void messageReceived(ShardRoutingEntry request, TransportChannel channel) throws Exception {
handleShardStartedOnMaster(request);
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}
}
class ShardStartedClusterStateHandler implements ClusterStateTaskExecutor<ShardRoutingEntry>, ClusterStateTaskListener {
@ -223,26 +262,20 @@ public class ShardStateAction extends AbstractComponent {
}
}
private class ShardFailedTransportHandler implements TransportRequestHandler<ShardRoutingEntry> {
private final ShardStartedClusterStateHandler shardStartedClusterStateHandler = new ShardStartedClusterStateHandler();
@Override
public void messageReceived(ShardRoutingEntry request, TransportChannel channel) throws Exception {
handleShardFailureOnMaster(request);
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}
}
private void handleShardStartedOnMaster(final ShardRoutingEntry shardRoutingEntry) {
logger.debug("received shard started for {}", shardRoutingEntry);
class ShardStartedTransportHandler implements TransportRequestHandler<ShardRoutingEntry> {
@Override
public void messageReceived(ShardRoutingEntry request, TransportChannel channel) throws Exception {
shardStartedOnMaster(request);
channel.sendResponse(TransportResponse.Empty.INSTANCE);
}
clusterService.submitStateUpdateTask(
"shard-started (" + shardRoutingEntry.shardRouting + "), reason [" + shardRoutingEntry.message + "]",
shardRoutingEntry,
ClusterStateTaskConfig.build(Priority.URGENT),
shardStartedClusterStateHandler,
shardStartedClusterStateHandler);
}
public static class ShardRoutingEntry extends TransportRequest {
ShardRouting shardRouting;
String indexUUID = IndexMetaData.INDEX_UUID_NA_VALUE;
String message;
@ -283,8 +316,13 @@ public class ShardStateAction extends AbstractComponent {
}
public interface Listener {
default void onSuccess() {}
default void onShardFailedNoMaster() {}
default void onShardFailedFailure(final DiscoveryNode master, final TransportException e) {}
default void onSuccess() {
}
default void onShardFailedNoMaster() {
}
default void onShardFailedFailure(final DiscoveryNode master, final TransportException e) {
}
}
}

2
core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
View File

@ -621,7 +621,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
public int numberOfReplicas() {
return settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1);
}
public Builder creationDate(long creationDate) {
settings = settingsBuilder().put(settings).put(SETTING_CREATION_DATE, creationDate).build();
return this;

32
core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
View File

@ -40,8 +40,8 @@ import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.support.LoggerMessageFormat;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.loader.SettingsLoader;
import org.elasticsearch.common.xcontent.FromXContentBuilder;
@ -134,13 +134,13 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
//noinspection unchecked
T proto = (T) customPrototypes.get(type);
if (proto == null) {
throw new IllegalArgumentException("No custom metadata prototype registered for type [" + type + "]");
throw new IllegalArgumentException("No custom metadata prototype registered for type [" + type + "], node likely missing plugins");
}
return proto;
}
public static final String SETTING_READ_ONLY = "cluster.blocks.read_only";
public static final Setting<Boolean> SETTING_READ_ONLY_SETTING = Setting.boolSetting("cluster.blocks.read_only", false, true, Setting.Scope.CLUSTER);
public static final ClusterBlock CLUSTER_READ_ONLY_BLOCK = new ClusterBlock(6, "cluster read-only (api)", false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA_WRITE));
@ -745,23 +745,23 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
/** All known byte-sized cluster settings. */
public static final Set<String> CLUSTER_BYTES_SIZE_SETTINGS = unmodifiableSet(newHashSet(
IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC,
RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC));
IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING.getKey(),
RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()));
/** All known time cluster settings. */
public static final Set<String> CLUSTER_TIME_SETTINGS = unmodifiableSet(newHashSet(
IndicesTTLService.INDICES_TTL_INTERVAL,
RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC,
RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK,
RecoverySettings.INDICES_RECOVERY_ACTIVITY_TIMEOUT,
RecoverySettings.INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT,
RecoverySettings.INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT,
DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL,
InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL,
InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT,
DiscoverySettings.PUBLISH_TIMEOUT,
InternalClusterService.SETTING_CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD));
IndicesTTLService.INDICES_TTL_INTERVAL_SETTING.getKey(),
RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING.getKey(),
RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING.getKey(),
RecoverySettings.INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING.getKey(),
RecoverySettings.INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING.getKey(),
RecoverySettings.INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING.getKey(),
DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(),
InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.getKey(),
InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(),
DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(),
InternalClusterService.CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.getKey()));
/** As of 2.0 we require units for time and byte-sized settings. This methods adds default units to any cluster settings that don't
* specify a unit. */

9
core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java
View File

@ -47,7 +47,6 @@ import org.elasticsearch.rest.RestStatus;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
/**
* Service responsible for submitting open/close index requests
@ -92,14 +91,6 @@ public class MetaDataIndexStateService extends AbstractComponent {
}
if (indexMetaData.getState() != IndexMetaData.State.CLOSE) {
IndexRoutingTable indexRoutingTable = currentState.routingTable().index(index);
for (IndexShardRoutingTable shard : indexRoutingTable) {
for (ShardRouting shardRouting : shard) {
if (shardRouting.primary() == true && shardRouting.allocatedPostIndexCreate() == false) {
throw new IndexPrimaryShardNotAllocatedException(new Index(index));
}
}
}
indicesToClose.add(index);
}
}

45
core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
View File

@ -37,7 +37,6 @@ import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.NodeServicesProvider;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.MergeResult;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.InvalidTypeNameException;
import org.elasticsearch.percolator.PercolatorService;
@ -237,8 +236,8 @@ public class MetaDataMappingService extends AbstractComponent {
}
private ClusterState applyRequest(ClusterState currentState, PutMappingClusterStateUpdateRequest request) throws IOException {
Map<String, DocumentMapper> newMappers = new HashMap<>();
Map<String, DocumentMapper> existingMappers = new HashMap<>();
String mappingType = request.type();
CompressedXContent mappingUpdateSource = new CompressedXContent(request.source());
for (String index : request.indices()) {
IndexService indexService = indicesService.indexServiceSafe(index);
// try and parse it (no need to add it here) so we can bail early in case of parsing exception
@ -246,16 +245,13 @@ public class MetaDataMappingService extends AbstractComponent {
DocumentMapper existingMapper = indexService.mapperService().documentMapper(request.type());
if (MapperService.DEFAULT_MAPPING.equals(request.type())) {
// _default_ types do not go through merging, but we do test the new settings. Also don't apply the old default
newMapper = indexService.mapperService().parse(request.type(), new CompressedXContent(request.source()), false);
newMapper = indexService.mapperService().parse(request.type(), mappingUpdateSource, false);
} else {
newMapper = indexService.mapperService().parse(request.type(), new CompressedXContent(request.source()), existingMapper == null);
newMapper = indexService.mapperService().parse(request.type(), mappingUpdateSource, existingMapper == null);
if (existingMapper != null) {
// first, simulate
MergeResult mergeResult = existingMapper.merge(newMapper.mapping(), true, request.updateAllTypes());
// if we have conflicts, throw an exception
if (mergeResult.hasConflicts()) {
throw new IllegalArgumentException("Merge failed with failures {" + Arrays.toString(mergeResult.buildConflicts()) + "}");
}
// this will just throw exceptions in case of problems
existingMapper.merge(newMapper.mapping(), true, request.updateAllTypes());
} else {
// TODO: can we find a better place for this validation?
// The reason this validation is here is that the mapper service doesn't learn about
@ -274,36 +270,31 @@ public class MetaDataMappingService extends AbstractComponent {
}
}
}
newMappers.put(index, newMapper);
if (existingMapper != null) {
existingMappers.put(index, existingMapper);
if (mappingType == null) {
mappingType = newMapper.type();
} else if (mappingType.equals(newMapper.type()) == false) {
throw new InvalidTypeNameException("Type name provided does not match type name within mapping definition");
}
}
assert mappingType != null;
String mappingType = request.type();
if (mappingType == null) {
mappingType = newMappers.values().iterator().next().type();
} else if (!mappingType.equals(newMappers.values().iterator().next().type())) {
throw new InvalidTypeNameException("Type name provided does not match type name within mapping definition");
}
if (!MapperService.DEFAULT_MAPPING.equals(mappingType) && !PercolatorService.TYPE_NAME.equals(mappingType) && mappingType.charAt(0) == '_') {
throw new InvalidTypeNameException("Document mapping type name can't start with '_'");
}
final Map<String, MappingMetaData> mappings = new HashMap<>();
for (Map.Entry<String, DocumentMapper> entry : newMappers.entrySet()) {
String index = entry.getKey();
for (String index : request.indices()) {
// do the actual merge here on the master, and update the mapping source
DocumentMapper newMapper = entry.getValue();
IndexService indexService = indicesService.indexService(index);
if (indexService == null) {
continue;
}
CompressedXContent existingSource = null;
if (existingMappers.containsKey(entry.getKey())) {
existingSource = existingMappers.get(entry.getKey()).mappingSource();
DocumentMapper existingMapper = indexService.mapperService().documentMapper(mappingType);
if (existingMapper != null) {
existingSource = existingMapper.mappingSource();
}
DocumentMapper mergedMapper = indexService.mapperService().merge(newMapper.type(), newMapper.mappingSource(), false, request.updateAllTypes());
DocumentMapper mergedMapper = indexService.mapperService().merge(mappingType, mappingUpdateSource, true, request.updateAllTypes());
CompressedXContent updatedSource = mergedMapper.mappingSource();
if (existingSource != null) {
@ -322,9 +313,9 @@ public class MetaDataMappingService extends AbstractComponent {
} else {
mappings.put(index, new MappingMetaData(mergedMapper));
if (logger.isDebugEnabled()) {
logger.debug("[{}] create_mapping [{}] with source [{}]", index, newMapper.type(), updatedSource);
logger.debug("[{}] create_mapping [{}] with source [{}]", index, mappingType, updatedSource);
} else if (logger.isInfoEnabled()) {
logger.info("[{}] create_mapping [{}]", index, newMapper.type());
logger.info("[{}] create_mapping [{}]", index, mappingType);
}
}
}

29
core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
View File

@ -21,12 +21,12 @@ package org.elasticsearch.cluster.routing;
import com.carrotsearch.hppc.ObjectIntHashMap;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import org.apache.lucene.util.CollectionUtil;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlocks;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.Randomness;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.index.shard.ShardId;
@ -34,7 +34,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@ -85,7 +84,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
Map<String, List<ShardRouting>> nodesToShards = new HashMap<>();
// fill in the nodeToShards with the "live" nodes
for (ObjectCursor<DiscoveryNode> cursor : clusterState.nodes().dataNodes().values()) {
nodesToShards.put(cursor.value.id(), new ArrayList<ShardRouting>());
nodesToShards.put(cursor.value.id(), new ArrayList<>());
}
// fill in the inverse of node -> shards allocated
@ -98,21 +97,13 @@ public class RoutingNodes implements Iterable<RoutingNode> {
// by the ShardId, as this is common for primary and replicas.
// A replica Set might have one (and not more) replicas with the state of RELOCATING.
if (shard.assignedToNode()) {
List<ShardRouting> entries = nodesToShards.get(shard.currentNodeId());
if (entries == null) {
entries = new ArrayList<>();
nodesToShards.put(shard.currentNodeId(), entries);
}
List<ShardRouting> entries = nodesToShards.computeIfAbsent(shard.currentNodeId(), k -> new ArrayList<>());
final ShardRouting sr = getRouting(shard, readOnly);
entries.add(sr);
assignedShardsAdd(sr);
if (shard.relocating()) {
entries = nodesToShards.get(shard.relocatingNodeId());
relocatingShards++;
if (entries == null) {
entries = new ArrayList<>();
nodesToShards.put(shard.relocatingNodeId(), entries);
}
entries = nodesToShards.computeIfAbsent(shard.relocatingNodeId(), k -> new ArrayList<>());
// add the counterpart shard with relocatingNodeId reflecting the source from which
// it's relocating from.
ShardRouting targetShardRouting = shard.buildTargetRelocatingShard();
@ -128,7 +119,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
inactiveShardCount++;
}
} else {
final ShardRouting sr = getRouting(shard, readOnly);
final ShardRouting sr = getRouting(shard, readOnly);
assignedShardsAdd(sr);
unassignedShards.add(sr);
}
@ -456,12 +447,8 @@ public class RoutingNodes implements Iterable<RoutingNode> {
// no unassigned
return;
}
List<ShardRouting> shards = assignedShards.get(shard.shardId());
if (shards == null) {
shards = new ArrayList<>();
assignedShards.put(shard.shardId(), shards);
}
assert assertInstanceNotInList(shard, shards);
List<ShardRouting> shards = assignedShards.computeIfAbsent(shard.shardId(), k -> new ArrayList<>());
assert assertInstanceNotInList(shard, shards);
shards.add(shard);
}
@ -671,7 +658,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
}
public void shuffle() {
Collections.shuffle(unassigned);
Randomness.shuffle(unassigned);
}
/**

9
core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
View File

@ -19,6 +19,8 @@
package org.elasticsearch.cluster.routing;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -267,7 +269,7 @@ public final class ShardRouting implements Streamable, ToXContent {
return shardIdentifier;
}
public boolean allocatedPostIndexCreate() {
public boolean allocatedPostIndexCreate(IndexMetaData indexMetaData) {
if (active()) {
return true;
}
@ -279,6 +281,11 @@ public final class ShardRouting implements Streamable, ToXContent {
return false;
}
if (indexMetaData.activeAllocationIds(id()).isEmpty() && indexMetaData.getCreationVersion().onOrAfter(Version.V_3_0_0)) {
// when no shards with this id have ever been active for this index
return false;
}
return true;
}

7
core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
View File

@ -39,15 +39,12 @@ import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocators;
import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
import org.elasticsearch.cluster.routing.allocation.decider.Decision;
import org.elasticsearch.common.Randomness;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;

51
core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
View File

@ -34,12 +34,13 @@ import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation;
import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
import org.elasticsearch.cluster.routing.allocation.decider.Decision;
import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.gateway.PriorityComparator;
import org.elasticsearch.node.settings.NodeSettingsService;
import java.util.ArrayList;
import java.util.Collection;
@ -72,42 +73,32 @@ import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
*/
public class BalancedShardsAllocator extends AbstractComponent implements ShardsAllocator {
public static final String SETTING_THRESHOLD = "cluster.routing.allocation.balance.threshold";
public static final String SETTING_INDEX_BALANCE_FACTOR = "cluster.routing.allocation.balance.index";
public static final String SETTING_SHARD_BALANCE_FACTOR = "cluster.routing.allocation.balance.shard";
private static final float DEFAULT_INDEX_BALANCE_FACTOR = 0.55f;
private static final float DEFAULT_SHARD_BALANCE_FACTOR = 0.45f;
class ApplySettings implements NodeSettingsService.Listener {
@Override
public void onRefreshSettings(Settings settings) {
final float indexBalance = settings.getAsFloat(SETTING_INDEX_BALANCE_FACTOR, weightFunction.indexBalance);
final float shardBalance = settings.getAsFloat(SETTING_SHARD_BALANCE_FACTOR, weightFunction.shardBalance);
float threshold = settings.getAsFloat(SETTING_THRESHOLD, BalancedShardsAllocator.this.threshold);
if (threshold <= 0.0f) {
throw new IllegalArgumentException("threshold must be greater than 0.0f but was: " + threshold);
}
BalancedShardsAllocator.this.threshold = threshold;
BalancedShardsAllocator.this.weightFunction = new WeightFunction(indexBalance, shardBalance);
}
}
private volatile WeightFunction weightFunction = new WeightFunction(DEFAULT_INDEX_BALANCE_FACTOR, DEFAULT_SHARD_BALANCE_FACTOR);
private volatile float threshold = 1.0f;
public static final Setting<Float> INDEX_BALANCE_FACTOR_SETTING = Setting.floatSetting("cluster.routing.allocation.balance.index", 0.55f, true, Setting.Scope.CLUSTER);
public static final Setting<Float> SHARD_BALANCE_FACTOR_SETTING = Setting.floatSetting("cluster.routing.allocation.balance.shard", 0.45f, true, Setting.Scope.CLUSTER);
public static final Setting<Float> THRESHOLD_SETTING = Setting.floatSetting("cluster.routing.allocation.balance.threshold", 1.0f, 0.0f, true, Setting.Scope.CLUSTER);
private volatile WeightFunction weightFunction;
private volatile float threshold;
public BalancedShardsAllocator(Settings settings) {
this(settings, new NodeSettingsService(settings));
this(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
}
@Inject
public BalancedShardsAllocator(Settings settings, NodeSettingsService nodeSettingsService) {
public BalancedShardsAllocator(Settings settings, ClusterSettings clusterSettings) {
super(settings);
ApplySettings applySettings = new ApplySettings();
applySettings.onRefreshSettings(settings);
nodeSettingsService.addListener(applySettings);
setWeightFunction(INDEX_BALANCE_FACTOR_SETTING.get(settings), SHARD_BALANCE_FACTOR_SETTING.get(settings));
setThreshold(THRESHOLD_SETTING.get(settings));
clusterSettings.addSettingsUpdateConsumer(INDEX_BALANCE_FACTOR_SETTING, SHARD_BALANCE_FACTOR_SETTING, this::setWeightFunction);
clusterSettings.addSettingsUpdateConsumer(THRESHOLD_SETTING, this::setThreshold);
}
private void setWeightFunction(float indexBalance, float shardBalanceFactor) {
weightFunction = new WeightFunction(indexBalance, shardBalanceFactor);
}
private void setThreshold(float threshold) {
this.threshold = threshold;
}
@Override

56
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
View File

@ -24,10 +24,11 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.settings.NodeSettingsService;
import java.util.HashMap;
import java.util.Map;
@ -76,37 +77,12 @@ public class AwarenessAllocationDecider extends AllocationDecider {
public static final String NAME = "awareness";
public static final String CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTES = "cluster.routing.allocation.awareness.attributes";
public static final String CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP = "cluster.routing.allocation.awareness.force.";
class ApplySettings implements NodeSettingsService.Listener {
@Override
public void onRefreshSettings(Settings settings) {
String[] awarenessAttributes = settings.getAsArray(CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTES, null);
if (awarenessAttributes == null && "".equals(settings.get(CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTES, null))) {
awarenessAttributes = Strings.EMPTY_ARRAY; // the empty string resets this
}
if (awarenessAttributes != null) {
logger.info("updating [cluster.routing.allocation.awareness.attributes] from [{}] to [{}]", AwarenessAllocationDecider.this.awarenessAttributes, awarenessAttributes);
AwarenessAllocationDecider.this.awarenessAttributes = awarenessAttributes;
}
Map<String, String[]> forcedAwarenessAttributes = new HashMap<>(AwarenessAllocationDecider.this.forcedAwarenessAttributes);
Map<String, Settings> forceGroups = settings.getGroups(CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP);
if (!forceGroups.isEmpty()) {
for (Map.Entry<String, Settings> entry : forceGroups.entrySet()) {
String[] aValues = entry.getValue().getAsArray("values");
if (aValues.length > 0) {
forcedAwarenessAttributes.put(entry.getKey(), aValues);
}
}
}
AwarenessAllocationDecider.this.forcedAwarenessAttributes = forcedAwarenessAttributes;
}
}
public static final Setting<String[]> CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING = new Setting<>("cluster.routing.allocation.awareness.attributes", "", Strings::splitStringByCommaToArray , true, Setting.Scope.CLUSTER);
public static final Setting<Settings> CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING = Setting.groupSetting("cluster.routing.allocation.awareness.force.", true, Setting.Scope.CLUSTER);
private String[] awarenessAttributes;
private Map<String, String[]> forcedAwarenessAttributes;
private volatile Map<String, String[]> forcedAwarenessAttributes;
/**
* Creates a new {@link AwarenessAllocationDecider} instance
@ -121,24 +97,28 @@ public class AwarenessAllocationDecider extends AllocationDecider {
* @param settings {@link Settings} to use
*/
public AwarenessAllocationDecider(Settings settings) {
this(settings, new NodeSettingsService(settings));
this(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
}
@Inject
public AwarenessAllocationDecider(Settings settings, NodeSettingsService nodeSettingsService) {
public AwarenessAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
super(settings);
this.awarenessAttributes = settings.getAsArray(CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTES);
this.awarenessAttributes = CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.get(settings);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING, this::setAwarenessAttributes);
setForcedAwarenessAttributes(CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.get(settings));
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING, this::setForcedAwarenessAttributes);
}
forcedAwarenessAttributes = new HashMap<>();
Map<String, Settings> forceGroups = settings.getGroups(CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP);
private void setForcedAwarenessAttributes(Settings forceSettings) {
Map<String, String[]> forcedAwarenessAttributes = new HashMap<>();
Map<String, Settings> forceGroups = forceSettings.getAsGroups();
for (Map.Entry<String, Settings> entry : forceGroups.entrySet()) {
String[] aValues = entry.getValue().getAsArray("values");
if (aValues.length > 0) {
forcedAwarenessAttributes.put(entry.getKey(), aValues);
}
}
nodeSettingsService.addListener(new ApplySettings());
this.forcedAwarenessAttributes = forcedAwarenessAttributes;
}
/**
@ -150,6 +130,10 @@ public class AwarenessAllocationDecider extends AllocationDecider {
return this.awarenessAttributes;
}
private void setAwarenessAttributes(String[] awarenessAttributes) {
this.awarenessAttributes = awarenessAttributes;
}
@Override
public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
return underCapacity(shardRouting, node, allocation, true);

62
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java
View File

@ -19,13 +19,12 @@
package org.elasticsearch.cluster.routing.allocation.decider;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
import org.elasticsearch.cluster.settings.Validator;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.settings.NodeSettingsService;
import java.util.Locale;
@ -38,10 +37,10 @@ import java.util.Locale;
* <ul>
* <li><tt>indices_primaries_active</tt> - Re-balancing is allowed only once all
* primary shards on all indices are active.</li>
*
*
* <li><tt>indices_all_active</tt> - Re-balancing is allowed only once all
* shards on all indices are active.</li>
*
*
* <li><tt>always</tt> - Re-balancing is allowed once a shard replication group
* is active</li>
* </ul>
@ -49,19 +48,10 @@ import java.util.Locale;
public class ClusterRebalanceAllocationDecider extends AllocationDecider {
public static final String NAME = "cluster_rebalance";
public static final String CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE = "cluster.routing.allocation.allow_rebalance";
public static final Validator ALLOCATION_ALLOW_REBALANCE_VALIDATOR = (setting, value, clusterState) -> {
try {
ClusterRebalanceType.parseString(value);
return null;
} catch (IllegalArgumentException e) {
return "the value of " + setting + " must be one of: [always, indices_primaries_active, indices_all_active]";
}
};
public static final Setting<ClusterRebalanceType> CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING = new Setting<>("cluster.routing.allocation.allow_rebalance", ClusterRebalanceType.INDICES_ALL_ACTIVE.name().toLowerCase(Locale.ROOT), ClusterRebalanceType::parseString, true, Setting.Scope.CLUSTER);
/**
* An enum representation for the configured re-balance type.
* An enum representation for the configured re-balance type.
*/
public static enum ClusterRebalanceType {
/**
@ -73,7 +63,7 @@ public class ClusterRebalanceAllocationDecider extends AllocationDecider {
*/
INDICES_PRIMARIES_ACTIVE,
/**
* Re-balancing is allowed only once all shards on all indices are active.
* Re-balancing is allowed only once all shards on all indices are active.
*/
INDICES_ALL_ACTIVE;
@ -85,48 +75,28 @@ public class ClusterRebalanceAllocationDecider extends AllocationDecider {
} else if ("indices_all_active".equalsIgnoreCase(typeString) || "indicesAllActive".equalsIgnoreCase(typeString)) {
return ClusterRebalanceType.INDICES_ALL_ACTIVE;
}
throw new IllegalArgumentException("Illegal value for " + CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE + ": " + typeString);
throw new IllegalArgumentException("Illegal value for " + CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING + ": " + typeString);
}
}
private ClusterRebalanceType type;
private volatile ClusterRebalanceType type;
@Inject
public ClusterRebalanceAllocationDecider(Settings settings, NodeSettingsService nodeSettingsService) {
public ClusterRebalanceAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
super(settings);
String allowRebalance = settings.get(CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "indices_all_active");
try {
type = ClusterRebalanceType.parseString(allowRebalance);
type = CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.get(settings);
} catch (IllegalStateException e) {
logger.warn("[{}] has a wrong value {}, defaulting to 'indices_all_active'", CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, allowRebalance);
logger.warn("[{}] has a wrong value {}, defaulting to 'indices_all_active'", CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING, CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getRaw(settings));
type = ClusterRebalanceType.INDICES_ALL_ACTIVE;
}
logger.debug("using [{}] with [{}]", CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, type.toString().toLowerCase(Locale.ROOT));
logger.debug("using [{}] with [{}]", CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING, type.toString().toLowerCase(Locale.ROOT));
nodeSettingsService.addListener(new ApplySettings());
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING, this::setType);
}
class ApplySettings implements NodeSettingsService.Listener {
@Override
public void onRefreshSettings(Settings settings) {
String newAllowRebalance = settings.get(CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, null);
if (newAllowRebalance != null) {
ClusterRebalanceType newType = null;
try {
newType = ClusterRebalanceType.parseString(newAllowRebalance);
} catch (IllegalArgumentException e) {
// ignore
}
if (newType != null && newType != ClusterRebalanceAllocationDecider.this.type) {
logger.info("updating [{}] from [{}] to [{}]", CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE,
ClusterRebalanceAllocationDecider.this.type.toString().toLowerCase(Locale.ROOT),
newType.toString().toLowerCase(Locale.ROOT));
ClusterRebalanceAllocationDecider.this.type = newType;
}
}
}
private void setType(ClusterRebalanceType type) {
this.type = type;
}
@Override

27
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java
View File

@ -22,8 +22,9 @@ package org.elasticsearch.cluster.routing.allocation.decider;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.settings.NodeSettingsService;
/**
* Similar to the {@link ClusterRebalanceAllocationDecider} this
@ -41,27 +42,19 @@ public class ConcurrentRebalanceAllocationDecider extends AllocationDecider {
public static final String NAME = "concurrent_rebalance";
public static final String CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE = "cluster.routing.allocation.cluster_concurrent_rebalance";
class ApplySettings implements NodeSettingsService.Listener {
@Override
public void onRefreshSettings(Settings settings) {
int clusterConcurrentRebalance = settings.getAsInt(CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE, ConcurrentRebalanceAllocationDecider.this.clusterConcurrentRebalance);
if (clusterConcurrentRebalance != ConcurrentRebalanceAllocationDecider.this.clusterConcurrentRebalance) {
logger.info("updating [cluster.routing.allocation.cluster_concurrent_rebalance] from [{}], to [{}]", ConcurrentRebalanceAllocationDecider.this.clusterConcurrentRebalance, clusterConcurrentRebalance);
ConcurrentRebalanceAllocationDecider.this.clusterConcurrentRebalance = clusterConcurrentRebalance;
}
}
}
public static final Setting<Integer> CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING = Setting.intSetting("cluster.routing.allocation.cluster_concurrent_rebalance", 2, -1, true, Setting.Scope.CLUSTER);
private volatile int clusterConcurrentRebalance;
@Inject
public ConcurrentRebalanceAllocationDecider(Settings settings, NodeSettingsService nodeSettingsService) {
public ConcurrentRebalanceAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
super(settings);
this.clusterConcurrentRebalance = settings.getAsInt(CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE, 2);
this.clusterConcurrentRebalance = CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING.get(settings);
logger.debug("using [cluster_concurrent_rebalance] with [{}]", clusterConcurrentRebalance);
nodeSettingsService.addListener(new ApplySettings());
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING, this::setClusterConcurrentRebalance);
}
private void setClusterConcurrentRebalance(int concurrentRebalance) {
clusterConcurrentRebalance = concurrentRebalance;
}
@Override

132
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
View File

@ -22,26 +22,27 @@ package org.elasticsearch.cluster.routing.allocation.decider;
import com.carrotsearch.hppc.ObjectLookupContainer;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterInfo;
import org.elasticsearch.cluster.ClusterInfoService;
import org.elasticsearch.cluster.DiskUsage;
import org.elasticsearch.cluster.EmptyClusterInfoService;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingState;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.RatioValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.node.settings.NodeSettingsService;
import java.util.Set;
@ -80,53 +81,11 @@ public class DiskThresholdDecider extends AllocationDecider {
private volatile boolean enabled;
private volatile TimeValue rerouteInterval;
public static final String CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED = "cluster.routing.allocation.disk.threshold_enabled";
public static final String CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK = "cluster.routing.allocation.disk.watermark.low";
public static final String CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK = "cluster.routing.allocation.disk.watermark.high";
public static final String CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS = "cluster.routing.allocation.disk.include_relocations";
public static final String CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL = "cluster.routing.allocation.disk.reroute_interval";
class ApplySettings implements NodeSettingsService.Listener {
@Override
public void onRefreshSettings(Settings settings) {
String newLowWatermark = settings.get(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK, null);
String newHighWatermark = settings.get(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK, null);
Boolean newRelocationsSetting = settings.getAsBoolean(CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS, null);
Boolean newEnableSetting = settings.getAsBoolean(CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED, null);
TimeValue newRerouteInterval = settings.getAsTime(CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL, null);
if (newEnableSetting != null) {
logger.info("updating [{}] from [{}] to [{}]", CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED,
DiskThresholdDecider.this.enabled, newEnableSetting);
DiskThresholdDecider.this.enabled = newEnableSetting;
}
if (newRelocationsSetting != null) {
logger.info("updating [{}] from [{}] to [{}]", CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS,
DiskThresholdDecider.this.includeRelocations, newRelocationsSetting);
DiskThresholdDecider.this.includeRelocations = newRelocationsSetting;
}
if (newLowWatermark != null) {
if (!validWatermarkSetting(newLowWatermark, CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK)) {
throw new ElasticsearchParseException("unable to parse low watermark [{}]", newLowWatermark);
}
logger.info("updating [{}] to [{}]", CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK, newLowWatermark);
DiskThresholdDecider.this.freeDiskThresholdLow = 100.0 - thresholdPercentageFromWatermark(newLowWatermark);
DiskThresholdDecider.this.freeBytesThresholdLow = thresholdBytesFromWatermark(newLowWatermark, CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK);
}
if (newHighWatermark != null) {
if (!validWatermarkSetting(newHighWatermark, CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK)) {
throw new ElasticsearchParseException("unable to parse high watermark [{}]", newHighWatermark);
}
logger.info("updating [{}] to [{}]", CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK, newHighWatermark);
DiskThresholdDecider.this.freeDiskThresholdHigh = 100.0 - thresholdPercentageFromWatermark(newHighWatermark);
DiskThresholdDecider.this.freeBytesThresholdHigh = thresholdBytesFromWatermark(newHighWatermark, CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK);
}
if (newRerouteInterval != null) {
logger.info("updating [{}] to [{}]", CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL, newRerouteInterval);
DiskThresholdDecider.this.rerouteInterval = newRerouteInterval;
}
}
}
public static final Setting<Boolean> CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING = Setting.boolSetting("cluster.routing.allocation.disk.threshold_enabled", true, true, Setting.Scope.CLUSTER);
public static final Setting<String> CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING = new Setting<>("cluster.routing.allocation.disk.watermark.low", "85%", (s) -> validWatermarkSetting(s, "cluster.routing.allocation.disk.watermark.low"), true, Setting.Scope.CLUSTER);
public static final Setting<String> CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING = new Setting<>("cluster.routing.allocation.disk.watermark.high", "90%", (s) -> validWatermarkSetting(s, "cluster.routing.allocation.disk.watermark.high"), true, Setting.Scope.CLUSTER);
public static final Setting<Boolean> CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS_SETTING = Setting.boolSetting("cluster.routing.allocation.disk.include_relocations", true, true, Setting.Scope.CLUSTER);;
public static final Setting<TimeValue> CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING = Setting.positiveTimeSetting("cluster.routing.allocation.disk.reroute_interval", TimeValue.timeValueSeconds(60), true, Setting.Scope.CLUSTER);
/**
* Listens for a node to go over the high watermark and kicks off an empty
@ -231,38 +190,49 @@ public class DiskThresholdDecider extends AllocationDecider {
// It's okay the Client is null here, because the empty cluster info
// service will never actually call the listener where the client is
// needed. Also this constructor is only used for tests
this(settings, new NodeSettingsService(settings), EmptyClusterInfoService.INSTANCE, null);
this(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), EmptyClusterInfoService.INSTANCE, null);
}
@Inject
public DiskThresholdDecider(Settings settings, NodeSettingsService nodeSettingsService, ClusterInfoService infoService, Client client) {
public DiskThresholdDecider(Settings settings, ClusterSettings clusterSettings, ClusterInfoService infoService, Client client) {
super(settings);
String lowWatermark = settings.get(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK, "85%");
String highWatermark = settings.get(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK, "90%");
if (!validWatermarkSetting(lowWatermark, CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK)) {
throw new ElasticsearchParseException("unable to parse low watermark [{}]", lowWatermark);
}
if (!validWatermarkSetting(highWatermark, CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK)) {
throw new ElasticsearchParseException("unable to parse high watermark [{}]", highWatermark);
}
// Watermark is expressed in terms of used data, but we need "free" data watermark
this.freeDiskThresholdLow = 100.0 - thresholdPercentageFromWatermark(lowWatermark);
this.freeDiskThresholdHigh = 100.0 - thresholdPercentageFromWatermark(highWatermark);
this.freeBytesThresholdLow = thresholdBytesFromWatermark(lowWatermark, CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK);
this.freeBytesThresholdHigh = thresholdBytesFromWatermark(highWatermark, CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK);
this.includeRelocations = settings.getAsBoolean(CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS, true);
this.rerouteInterval = settings.getAsTime(CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL, TimeValue.timeValueSeconds(60));
this.enabled = settings.getAsBoolean(CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED, true);
nodeSettingsService.addListener(new ApplySettings());
final String lowWatermark = CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.get(settings);
final String highWatermark = CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.get(settings);
setHighWatermark(highWatermark);
setLowWatermark(lowWatermark);
this.includeRelocations = CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS_SETTING.get(settings);
this.rerouteInterval = CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.get(settings);
this.enabled = CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.get(settings);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING, this::setLowWatermark);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING, this::setHighWatermark);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS_SETTING, this::setIncludeRelocations);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING, this::setRerouteInterval);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING, this::setEnabled);
infoService.addListener(new DiskListener(client));
}
// For Testing
ApplySettings newApplySettings() {
return new ApplySettings();
private void setIncludeRelocations(boolean includeRelocations) {
this.includeRelocations = includeRelocations;
}
private void setRerouteInterval(TimeValue rerouteInterval) {
this.rerouteInterval = rerouteInterval;
}
private void setEnabled(boolean enabled) {
this.enabled = enabled;
}
private void setLowWatermark(String lowWatermark) {
// Watermark is expressed in terms of used data, but we need "free" data watermark
this.freeDiskThresholdLow = 100.0 - thresholdPercentageFromWatermark(lowWatermark);
this.freeBytesThresholdLow = thresholdBytesFromWatermark(lowWatermark, CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey());
}
private void setHighWatermark(String highWatermark) {
// Watermark is expressed in terms of used data, but we need "free" data watermark
this.freeDiskThresholdHigh = 100.0 - thresholdPercentageFromWatermark(highWatermark);
this.freeBytesThresholdHigh = thresholdBytesFromWatermark(highWatermark, CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey());
}
// For Testing
@ -360,7 +330,8 @@ public class DiskThresholdDecider extends AllocationDecider {
}
// a flag for whether the primary shard has been previously allocated
boolean primaryHasBeenAllocated = shardRouting.primary() && shardRouting.allocatedPostIndexCreate();
IndexMetaData indexMetaData = allocation.metaData().index(shardRouting.getIndex());
boolean primaryHasBeenAllocated = shardRouting.primary() && shardRouting.allocatedPostIndexCreate(indexMetaData);
// checks for exact byte comparisons
if (freeBytes < freeBytesThresholdLow.bytes()) {
@ -580,20 +551,21 @@ public class DiskThresholdDecider extends AllocationDecider {
/**
* Checks if a watermark string is a valid percentage or byte size value,
* returning true if valid, false if invalid.
* @return the watermark value given
*/
public boolean validWatermarkSetting(String watermark, String settingName) {
public static String validWatermarkSetting(String watermark, String settingName) {
try {
RatioValue.parseRatioValue(watermark);
return true;
} catch (ElasticsearchParseException e) {
try {
ByteSizeValue.parseBytesSizeValue(watermark, settingName);
return true;
} catch (ElasticsearchParseException ex) {
return false;
ex.addSuppressed(e);
throw ex;
}
}
return watermark;
}
private Decision earlyTerminate(RoutingAllocation allocation, ImmutableOpenMap<String, DiskUsage> usages) {

60
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
View File

@ -19,18 +19,20 @@
package org.elasticsearch.cluster.routing.allocation.decider;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.settings.NodeSettingsService;
import java.util.Locale;
/**
* This allocation decider allows shard allocations / rebalancing via the cluster wide settings {@link #CLUSTER_ROUTING_ALLOCATION_ENABLE} /
* {@link #CLUSTER_ROUTING_REBALANCE_ENABLE} and the per index setting {@link #INDEX_ROUTING_ALLOCATION_ENABLE} / {@link #INDEX_ROUTING_REBALANCE_ENABLE}.
* This allocation decider allows shard allocations / rebalancing via the cluster wide settings {@link #CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING} /
* {@link #CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING} and the per index setting {@link #INDEX_ROUTING_ALLOCATION_ENABLE} / {@link #INDEX_ROUTING_REBALANCE_ENABLE}.
* The per index settings overrides the cluster wide setting.
*
* <p>
@ -54,26 +56,34 @@ import java.util.Locale;
* @see Rebalance
* @see Allocation
*/
public class EnableAllocationDecider extends AllocationDecider implements NodeSettingsService.Listener {
public class EnableAllocationDecider extends AllocationDecider {
public static final String NAME = "enable";
public static final String CLUSTER_ROUTING_ALLOCATION_ENABLE = "cluster.routing.allocation.enable";
public static final String INDEX_ROUTING_ALLOCATION_ENABLE = "index.routing.allocation.enable";
public static final Setting<Allocation> CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING = new Setting<>("cluster.routing.allocation.enable", Allocation.ALL.name(), Allocation::parse, true, Setting.Scope.CLUSTER);
public static final String INDEX_ROUTING_ALLOCATION_ENABLE= "index.routing.allocation.enable";
public static final String CLUSTER_ROUTING_REBALANCE_ENABLE = "cluster.routing.rebalance.enable";
public static final Setting<Rebalance> CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING = new Setting<>("cluster.routing.rebalance.enable", Rebalance.ALL.name(), Rebalance::parse, true, Setting.Scope.CLUSTER);
public static final String INDEX_ROUTING_REBALANCE_ENABLE = "index.routing.rebalance.enable";
private volatile Rebalance enableRebalance;
private volatile Allocation enableAllocation;
@Inject
public EnableAllocationDecider(Settings settings, NodeSettingsService nodeSettingsService) {
public EnableAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
super(settings);
this.enableAllocation = Allocation.parse(settings.get(CLUSTER_ROUTING_ALLOCATION_ENABLE, Allocation.ALL.name()));
this.enableRebalance = Rebalance.parse(settings.get(CLUSTER_ROUTING_REBALANCE_ENABLE, Rebalance.ALL.name()));
nodeSettingsService.addListener(this);
this.enableAllocation = CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.get(settings);
this.enableRebalance = CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.get(settings);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING, this::setEnableAllocation);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING, this::setEnableRebalance);
}
public void setEnableRebalance(Rebalance enableRebalance) {
this.enableRebalance = enableRebalance;
}
public void setEnableAllocation(Allocation enableAllocation) {
this.enableAllocation = enableAllocation;
}
@Override
@ -82,8 +92,8 @@ public class EnableAllocationDecider extends AllocationDecider implements NodeSe
return allocation.decision(Decision.YES, NAME, "allocation disabling is ignored");
}
Settings indexSettings = allocation.routingNodes().metaData().index(shardRouting.index()).getSettings();
String enableIndexValue = indexSettings.get(INDEX_ROUTING_ALLOCATION_ENABLE);
IndexMetaData indexMetaData = allocation.metaData().index(shardRouting.getIndex());
String enableIndexValue = indexMetaData.getSettings().get(INDEX_ROUTING_ALLOCATION_ENABLE);
final Allocation enable;
if (enableIndexValue != null) {
enable = Allocation.parse(enableIndexValue);
@ -96,7 +106,7 @@ public class EnableAllocationDecider extends AllocationDecider implements NodeSe
case NONE:
return allocation.decision(Decision.NO, NAME, "no allocations are allowed");
case NEW_PRIMARIES:
if (shardRouting.primary() && shardRouting.allocatedPostIndexCreate() == false) {
if (shardRouting.primary() && shardRouting.allocatedPostIndexCreate(indexMetaData) == false) {
return allocation.decision(Decision.YES, NAME, "new primary allocations are allowed");
} else {
return allocation.decision(Decision.NO, NAME, "non-new primary allocations are forbidden");
@ -148,25 +158,9 @@ public class EnableAllocationDecider extends AllocationDecider implements NodeSe
}
}
@Override
public void onRefreshSettings(Settings settings) {
final Allocation enable = Allocation.parse(settings.get(CLUSTER_ROUTING_ALLOCATION_ENABLE, this.enableAllocation.name()));
if (enable != this.enableAllocation) {
logger.info("updating [{}] from [{}] to [{}]", CLUSTER_ROUTING_ALLOCATION_ENABLE, this.enableAllocation, enable);
EnableAllocationDecider.this.enableAllocation = enable;
}
final Rebalance enableRebalance = Rebalance.parse(settings.get(CLUSTER_ROUTING_REBALANCE_ENABLE, this.enableRebalance.name()));
if (enableRebalance != this.enableRebalance) {
logger.info("updating [{}] from [{}] to [{}]", CLUSTER_ROUTING_REBALANCE_ENABLE, this.enableRebalance, enableRebalance);
EnableAllocationDecider.this.enableRebalance = enableRebalance;
}
}
/**
* Allocation values or rather their string representation to be used used with
* {@link EnableAllocationDecider#CLUSTER_ROUTING_ALLOCATION_ENABLE} / {@link EnableAllocationDecider#INDEX_ROUTING_ALLOCATION_ENABLE}
* {@link EnableAllocationDecider#CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING} / {@link EnableAllocationDecider#INDEX_ROUTING_ALLOCATION_ENABLE}
* via cluster / index settings.
*/
public enum Allocation {
@ -192,7 +186,7 @@ public class EnableAllocationDecider extends AllocationDecider implements NodeSe
/**
* Rebalance values or rather their string representation to be used used with
* {@link EnableAllocationDecider#CLUSTER_ROUTING_REBALANCE_ENABLE} / {@link EnableAllocationDecider#INDEX_ROUTING_REBALANCE_ENABLE}
* {@link EnableAllocationDecider#CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING} / {@link EnableAllocationDecider#INDEX_ROUTING_REBALANCE_ENABLE}
* via cluster / index settings.
*/
public enum Rebalance {

62
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
View File

@ -25,10 +25,9 @@ import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.settings.NodeSettingsService;
import java.util.Map;
import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.AND;
import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.OR;
@ -65,36 +64,23 @@ public class FilterAllocationDecider extends AllocationDecider {
public static final String INDEX_ROUTING_INCLUDE_GROUP = "index.routing.allocation.include.";
public static final String INDEX_ROUTING_EXCLUDE_GROUP = "index.routing.allocation.exclude.";
public static final String CLUSTER_ROUTING_REQUIRE_GROUP = "cluster.routing.allocation.require.";
public static final String CLUSTER_ROUTING_INCLUDE_GROUP = "cluster.routing.allocation.include.";
public static final String CLUSTER_ROUTING_EXCLUDE_GROUP = "cluster.routing.allocation.exclude.";
public static final Setting<Settings> CLUSTER_ROUTING_REQUIRE_GROUP_SETTING = Setting.groupSetting("cluster.routing.allocation.require.", true, Setting.Scope.CLUSTER);
public static final Setting<Settings> CLUSTER_ROUTING_INCLUDE_GROUP_SETTING = Setting.groupSetting("cluster.routing.allocation.include.", true, Setting.Scope.CLUSTER);
public static final Setting<Settings> CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING = Setting.groupSetting("cluster.routing.allocation.exclude.", true, Setting.Scope.CLUSTER);
private volatile DiscoveryNodeFilters clusterRequireFilters;
private volatile DiscoveryNodeFilters clusterIncludeFilters;
private volatile DiscoveryNodeFilters clusterExcludeFilters;
@Inject
public FilterAllocationDecider(Settings settings, NodeSettingsService nodeSettingsService) {
public FilterAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
super(settings);
Map<String, String> requireMap = settings.getByPrefix(CLUSTER_ROUTING_REQUIRE_GROUP).getAsMap();
if (requireMap.isEmpty()) {
clusterRequireFilters = null;
} else {
clusterRequireFilters = DiscoveryNodeFilters.buildFromKeyValue(AND, requireMap);
}
Map<String, String> includeMap = settings.getByPrefix(CLUSTER_ROUTING_INCLUDE_GROUP).getAsMap();
if (includeMap.isEmpty()) {
clusterIncludeFilters = null;
} else {
clusterIncludeFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, includeMap);
}
Map<String, String> excludeMap = settings.getByPrefix(CLUSTER_ROUTING_EXCLUDE_GROUP).getAsMap();
if (excludeMap.isEmpty()) {
clusterExcludeFilters = null;
} else {
clusterExcludeFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, excludeMap);
}
nodeSettingsService.addListener(new ApplySettings());
setClusterRequireFilters(CLUSTER_ROUTING_REQUIRE_GROUP_SETTING.get(settings));
setClusterExcludeFilters(CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING.get(settings));
setClusterIncludeFilters(CLUSTER_ROUTING_INCLUDE_GROUP_SETTING.get(settings));
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_REQUIRE_GROUP_SETTING, this::setClusterRequireFilters);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING, this::setClusterExcludeFilters);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_INCLUDE_GROUP_SETTING, this::setClusterIncludeFilters);
}
@Override
@ -144,21 +130,13 @@ public class FilterAllocationDecider extends AllocationDecider {
return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require filters");
}
class ApplySettings implements NodeSettingsService.Listener {
@Override
public void onRefreshSettings(Settings settings) {
Map<String, String> requireMap = settings.getByPrefix(CLUSTER_ROUTING_REQUIRE_GROUP).getAsMap();
if (!requireMap.isEmpty()) {
clusterRequireFilters = DiscoveryNodeFilters.buildFromKeyValue(AND, requireMap);
}
Map<String, String> includeMap = settings.getByPrefix(CLUSTER_ROUTING_INCLUDE_GROUP).getAsMap();
if (!includeMap.isEmpty()) {
clusterIncludeFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, includeMap);
}
Map<String, String> excludeMap = settings.getByPrefix(CLUSTER_ROUTING_EXCLUDE_GROUP).getAsMap();
if (!excludeMap.isEmpty()) {
clusterExcludeFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, excludeMap);
}
}
private void setClusterRequireFilters(Settings settings) {
clusterRequireFilters = DiscoveryNodeFilters.buildFromKeyValue(AND, settings.getAsMap());
}
private void setClusterIncludeFilters(Settings settings) {
clusterIncludeFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, settings.getAsMap());
}
private void setClusterExcludeFilters(Settings settings) {
clusterExcludeFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, settings.getAsMap());
}
}

30
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
View File

@ -24,16 +24,16 @@ import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingState;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.settings.NodeSettingsService;
/**
* This {@link AllocationDecider} limits the number of shards per node on a per
* index or node-wide basis. The allocator prevents a single node to hold more
* than {@value #INDEX_TOTAL_SHARDS_PER_NODE} per index and
* {@value #CLUSTER_TOTAL_SHARDS_PER_NODE} globally during the allocation
* <tt>cluster.routing.allocation.total_shards_per_node</tt> globally during the allocation
* process. The limits of this decider can be changed in real-time via a the
* index settings API.
* <p>
@ -64,26 +64,18 @@ public class ShardsLimitAllocationDecider extends AllocationDecider {
* Controls the maximum number of shards per node on a global level.
* Negative values are interpreted as unlimited.
*/
public static final String CLUSTER_TOTAL_SHARDS_PER_NODE = "cluster.routing.allocation.total_shards_per_node";
public static final Setting<Integer> CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING = Setting.intSetting("cluster.routing.allocation.total_shards_per_node", -1, true, Setting.Scope.CLUSTER);
class ApplySettings implements NodeSettingsService.Listener {
@Override
public void onRefreshSettings(Settings settings) {
Integer newClusterLimit = settings.getAsInt(CLUSTER_TOTAL_SHARDS_PER_NODE, null);
if (newClusterLimit != null) {
logger.info("updating [{}] from [{}] to [{}]", CLUSTER_TOTAL_SHARDS_PER_NODE,
ShardsLimitAllocationDecider.this.clusterShardLimit, newClusterLimit);
ShardsLimitAllocationDecider.this.clusterShardLimit = newClusterLimit;
}
}
}
@Inject
public ShardsLimitAllocationDecider(Settings settings, NodeSettingsService nodeSettingsService) {
public ShardsLimitAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
super(settings);
this.clusterShardLimit = settings.getAsInt(CLUSTER_TOTAL_SHARDS_PER_NODE, -1);
nodeSettingsService.addListener(new ApplySettings());
this.clusterShardLimit = CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.get(settings);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING, this::setClusterShardLimit);
}
private void setClusterShardLimit(int clusterShardLimit) {
this.clusterShardLimit = clusterShardLimit;
}
@Override

28
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java
View File

@ -23,9 +23,10 @@ import org.elasticsearch.cluster.SnapshotsInProgress;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.settings.NodeSettingsService;
/**
* This {@link org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider} prevents shards that
@ -38,18 +39,7 @@ public class SnapshotInProgressAllocationDecider extends AllocationDecider {
/**
* Disables relocation of shards that are currently being snapshotted.
*/
public static final String CLUSTER_ROUTING_ALLOCATION_SNAPSHOT_RELOCATION_ENABLED = "cluster.routing.allocation.snapshot.relocation_enabled";
class ApplySettings implements NodeSettingsService.Listener {
@Override
public void onRefreshSettings(Settings settings) {
boolean newEnableRelocation = settings.getAsBoolean(CLUSTER_ROUTING_ALLOCATION_SNAPSHOT_RELOCATION_ENABLED, enableRelocation);
if (newEnableRelocation != enableRelocation) {
logger.info("updating [{}] from [{}], to [{}]", CLUSTER_ROUTING_ALLOCATION_SNAPSHOT_RELOCATION_ENABLED, enableRelocation, newEnableRelocation);
enableRelocation = newEnableRelocation;
}
}
}
public static final Setting<Boolean> CLUSTER_ROUTING_ALLOCATION_SNAPSHOT_RELOCATION_ENABLED_SETTING = Setting.boolSetting("cluster.routing.allocation.snapshot.relocation_enabled", false, true, Setting.Scope.CLUSTER);
private volatile boolean enableRelocation = false;
@ -66,14 +56,18 @@ public class SnapshotInProgressAllocationDecider extends AllocationDecider {
* @param settings {@link org.elasticsearch.common.settings.Settings} to use
*/
public SnapshotInProgressAllocationDecider(Settings settings) {
this(settings, new NodeSettingsService(settings));
this(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
}
@Inject
public SnapshotInProgressAllocationDecider(Settings settings, NodeSettingsService nodeSettingsService) {
public SnapshotInProgressAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
super(settings);
enableRelocation = settings.getAsBoolean(CLUSTER_ROUTING_ALLOCATION_SNAPSHOT_RELOCATION_ENABLED, enableRelocation);
nodeSettingsService.addListener(new ApplySettings());
enableRelocation = CLUSTER_ROUTING_ALLOCATION_SNAPSHOT_RELOCATION_ENABLED_SETTING.get(settings);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_SNAPSHOT_RELOCATION_ENABLED_SETTING, this::setEnableRelocation);
}
private void setEnableRelocation(boolean enableRelocation) {
this.enableRelocation = enableRelocation;
}
/**

50
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
View File

@ -19,13 +19,14 @@
package org.elasticsearch.cluster.routing.allocation.decider;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingState;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.node.settings.NodeSettingsService;
/**
* {@link ThrottlingAllocationDecider} controls the recovery process per node in
@ -47,27 +48,33 @@ import org.elasticsearch.node.settings.NodeSettingsService;
*/
public class ThrottlingAllocationDecider extends AllocationDecider {
public static final String NAME = "throttling";
public static final String CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES = "cluster.routing.allocation.node_initial_primaries_recoveries";
public static final String CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES = "cluster.routing.allocation.node_concurrent_recoveries";
public static final String CLUSTER_ROUTING_ALLOCATION_CONCURRENT_RECOVERIES = "cluster.routing.allocation.concurrent_recoveries";
public static final int DEFAULT_CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES = 2;
public static final int DEFAULT_CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES = 4;
public static final String NAME = "throttling";
public static final String CLUSTER_ROUTING_ALLOCATION_CONCURRENT_RECOVERIES = "cluster.routing.allocation.concurrent_recoveries";
public static final Setting<Integer> CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING = Setting.intSetting("cluster.routing.allocation.node_initial_primaries_recoveries", DEFAULT_CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES, 0, true, Setting.Scope.CLUSTER);
public static final Setting<Integer> CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING = new Setting<>("cluster.routing.allocation.node_concurrent_recoveries", (s) -> s.get(CLUSTER_ROUTING_ALLOCATION_CONCURRENT_RECOVERIES,Integer.toString(DEFAULT_CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES)), (s) -> Setting.parseInt(s, 0, "cluster.routing.allocation.node_concurrent_recoveries"), true, Setting.Scope.CLUSTER);
private volatile int primariesInitialRecoveries;
private volatile int concurrentRecoveries;
@Inject
public ThrottlingAllocationDecider(Settings settings, NodeSettingsService nodeSettingsService) {
public ThrottlingAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
super(settings);
this.primariesInitialRecoveries = settings.getAsInt(CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES, DEFAULT_CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES);
this.concurrentRecoveries = settings.getAsInt(CLUSTER_ROUTING_ALLOCATION_CONCURRENT_RECOVERIES, settings.getAsInt(CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES, DEFAULT_CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES));
this.primariesInitialRecoveries = CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING.get(settings);
this.concurrentRecoveries = CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING.get(settings);
logger.debug("using node_concurrent_recoveries [{}], node_initial_primaries_recoveries [{}]", concurrentRecoveries, primariesInitialRecoveries);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING, this::setPrimariesInitialRecoveries);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING, this::setConcurrentRecoveries);
}
nodeSettingsService.addListener(new ApplySettings());
private void setConcurrentRecoveries(int concurrentRecoveries) {
this.concurrentRecoveries = concurrentRecoveries;
}
private void setPrimariesInitialRecoveries(int primariesInitialRecoveries) {
this.primariesInitialRecoveries = primariesInitialRecoveries;
}
@Override
@ -115,21 +122,4 @@ public class ThrottlingAllocationDecider extends AllocationDecider {
return allocation.decision(Decision.YES, NAME, "below shard recovery limit of [%d]", concurrentRecoveries);
}
}
class ApplySettings implements NodeSettingsService.Listener {
@Override
public void onRefreshSettings(Settings settings) {
int primariesInitialRecoveries = settings.getAsInt(CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES, ThrottlingAllocationDecider.this.primariesInitialRecoveries);
if (primariesInitialRecoveries != ThrottlingAllocationDecider.this.primariesInitialRecoveries) {
logger.info("updating [cluster.routing.allocation.node_initial_primaries_recoveries] from [{}] to [{}]", ThrottlingAllocationDecider.this.primariesInitialRecoveries, primariesInitialRecoveries);
ThrottlingAllocationDecider.this.primariesInitialRecoveries = primariesInitialRecoveries;
}
int concurrentRecoveries = settings.getAsInt(CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES, ThrottlingAllocationDecider.this.concurrentRecoveries);
if (concurrentRecoveries != ThrottlingAllocationDecider.this.concurrentRecoveries) {
logger.info("updating [cluster.routing.allocation.node_concurrent_recoveries] from [{}] to [{}]", ThrottlingAllocationDecider.this.concurrentRecoveries, concurrentRecoveries);
ThrottlingAllocationDecider.this.concurrentRecoveries = concurrentRecoveries;
}
}
}
}

102
core/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java
View File

@ -20,8 +20,19 @@
package org.elasticsearch.cluster.service;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.*;
import org.elasticsearch.cluster.AckedClusterStateTaskListener;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterState.Builder;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.ClusterStateTaskConfig;
import org.elasticsearch.cluster.ClusterStateTaskExecutor;
import org.elasticsearch.cluster.ClusterStateTaskListener;
import org.elasticsearch.cluster.ClusterStateUpdateTask;
import org.elasticsearch.cluster.LocalNodeMasterListener;
import org.elasticsearch.cluster.TimeoutClusterStateListener;
import org.elasticsearch.cluster.block.ClusterBlock;
import org.elasticsearch.cluster.block.ClusterBlocks;
import org.elasticsearch.cluster.metadata.MetaData;
@ -38,20 +49,39 @@ import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.text.StringText;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.*;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.common.util.concurrent.CountDown;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
import org.elasticsearch.common.util.concurrent.FutureUtils;
import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
import org.elasticsearch.common.util.concurrent.PrioritizedRunnable;
import org.elasticsearch.common.util.iterable.Iterables;
import org.elasticsearch.discovery.Discovery;
import org.elasticsearch.discovery.DiscoveryService;
import org.elasticsearch.node.settings.NodeSettingsService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.util.*;
import java.util.concurrent.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Executor;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
@ -62,8 +92,8 @@ import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadF
*/
public class InternalClusterService extends AbstractLifecycleComponent<ClusterService> implements ClusterService {
public static final String SETTING_CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD = "cluster.service.slow_task_logging_threshold";
public static final String SETTING_CLUSTER_SERVICE_RECONNECT_INTERVAL = "cluster.service.reconnect_interval";
public static final Setting<TimeValue> CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING = Setting.positiveTimeSetting("cluster.service.slow_task_logging_threshold", TimeValue.timeValueSeconds(30), true, Setting.Scope.CLUSTER);
public static final Setting<TimeValue> CLUSTER_SERVICE_RECONNECT_INTERVAL_SETTING = Setting.positiveTimeSetting("cluster.service.reconnect_interval", TimeValue.timeValueSeconds(10), false, Setting.Scope.CLUSTER);
public static final String UPDATE_THREAD_NAME = "clusterService#updateTask";
private final ThreadPool threadPool;
@ -74,7 +104,7 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
private final TransportService transportService;
private final NodeSettingsService nodeSettingsService;
private final ClusterSettings clusterSettings;
private final DiscoveryNodeService discoveryNodeService;
private final Version version;
@ -107,33 +137,32 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
@Inject
public InternalClusterService(Settings settings, DiscoveryService discoveryService, OperationRouting operationRouting, TransportService transportService,
NodeSettingsService nodeSettingsService, ThreadPool threadPool, ClusterName clusterName, DiscoveryNodeService discoveryNodeService, Version version) {
ClusterSettings clusterSettings, ThreadPool threadPool, ClusterName clusterName, DiscoveryNodeService discoveryNodeService, Version version) {
super(settings);
this.operationRouting = operationRouting;
this.transportService = transportService;
this.discoveryService = discoveryService;
this.threadPool = threadPool;
this.nodeSettingsService = nodeSettingsService;
this.clusterSettings = clusterSettings;
this.discoveryNodeService = discoveryNodeService;
this.version = version;
// will be replaced on doStart.
this.clusterState = ClusterState.builder(clusterName).build();
this.nodeSettingsService.setClusterService(this);
this.nodeSettingsService.addListener(new ApplySettings());
this.clusterSettings.addSettingsUpdateConsumer(CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING, this::setSlowTaskLoggingThreshold);
this.reconnectInterval = this.settings.getAsTime(SETTING_CLUSTER_SERVICE_RECONNECT_INTERVAL, TimeValue.timeValueSeconds(10));
this.reconnectInterval = CLUSTER_SERVICE_RECONNECT_INTERVAL_SETTING.get(settings);
this.slowTaskLoggingThreshold = this.settings.getAsTime(SETTING_CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD, TimeValue.timeValueSeconds(30));
this.slowTaskLoggingThreshold = CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get(settings);
localNodeMasterListeners = new LocalNodeMasterListeners(threadPool);
initialBlocks = ClusterBlocks.builder().addGlobalBlock(discoveryService.getNoMasterBlock());
}
public NodeSettingsService settingsService() {
return this.nodeSettingsService;
private void setSlowTaskLoggingThreshold(TimeValue slowTaskLoggingThreshold) {
this.slowTaskLoggingThreshold = slowTaskLoggingThreshold;
}
@Override
@ -292,6 +321,7 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
if (config.timeout() != null) {
updateTasksExecutor.execute(updateTask, threadPool.scheduler(), config.timeout(), () -> threadPool.generic().execute(() -> {
if (updateTask.processed.getAndSet(true) == false) {
logger.debug("cluster state update task [{}] timed out after [{}]", source, config.timeout());
listener.onFailure(source, new ProcessClusterEventTimeoutException(config.timeout(), source));
}}));
} else {
@ -327,7 +357,7 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
timeInQueue = 0;
}
pendingClusterTasks.add(new PendingClusterTask(pending.insertionOrder, pending.priority, new StringText(source), timeInQueue, pending.executing));
pendingClusterTasks.add(new PendingClusterTask(pending.insertionOrder, pending.priority, new Text(source), timeInQueue, pending.executing));
}
return pendingClusterTasks;
}
@ -413,6 +443,15 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
}
assert batchResult.executionResults != null;
assert batchResult.executionResults.size() == toExecute.size()
: String.format(Locale.ROOT, "expected [%d] task result%s but was [%d]", toExecute.size(), toExecute.size() == 1 ? "" : "s", batchResult.executionResults.size());
boolean assertsEnabled = false;
assert (assertsEnabled = true);
if (assertsEnabled) {
for (UpdateTask<T> updateTask : toExecute) {
assert batchResult.executionResults.containsKey(updateTask.task) : "missing task result for [" + updateTask.task + "]";
}
}
ClusterState newClusterState = batchResult.resultingState;
final ArrayList<UpdateTask<T>> proccessedListeners = new ArrayList<>();
@ -421,7 +460,13 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
assert batchResult.executionResults.containsKey(updateTask.task) : "missing " + updateTask.task.toString();
final ClusterStateTaskExecutor.TaskResult executionResult =
batchResult.executionResults.get(updateTask.task);
executionResult.handle(() -> proccessedListeners.add(updateTask), ex -> updateTask.listener.onFailure(updateTask.source, ex));
executionResult.handle(
() -> proccessedListeners.add(updateTask),
ex -> {
logger.debug("cluster state update task [{}] failed", ex, updateTask.source);
updateTask.listener.onFailure(updateTask.source, ex);
}
);
}
if (previousClusterState == newClusterState) {
@ -521,6 +566,15 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
// update the current cluster state
clusterState = newClusterState;
logger.debug("set local cluster state to version {}", newClusterState.version());
try {
// nothing to do until we actually recover from the gateway or any other block indicates we need to disable persistency
if (clusterChangedEvent.state().blocks().disableStatePersistence() == false && clusterChangedEvent.metaDataChanged()) {
final Settings incomingSettings = clusterChangedEvent.state().metaData().settings();
clusterSettings.applySettings(incomingSettings);
}
} catch (Exception ex) {
logger.warn("failed to apply cluster settings", ex);
}
for (ClusterStateListener listener : preAppliedListeners) {
try {
listener.clusterChanged(clusterChangedEvent);
@ -560,6 +614,8 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
task.listener.clusterStateProcessed(task.source, previousClusterState, newClusterState);
}
executor.clusterStatePublished(newClusterState);
TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - startTimeNS)));
logger.debug("processing [{}]: took {} done applying updated cluster_state (version: {}, uuid: {})", source, executionTime, newClusterState.version(), newClusterState.stateUUID());
warnAboutSlowTaskIfNeeded(executionTime, source);
@ -846,12 +902,4 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
}
}
}
class ApplySettings implements NodeSettingsService.Listener {
@Override
public void onRefreshSettings(Settings settings) {
final TimeValue slowTaskLoggingThreshold = settings.getAsTime(SETTING_CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD, InternalClusterService.this.slowTaskLoggingThreshold);
InternalClusterService.this.slowTaskLoggingThreshold = slowTaskLoggingThreshold;
}
}
}

3
core/src/main/java/org/elasticsearch/common/Booleans.java
View File

@ -84,7 +84,6 @@ public class Booleans {
* throws exception if string cannot be parsed to boolean
*/
public static Boolean parseBooleanExact(String value) {
boolean isFalse = isExplicitFalse(value);
if (isFalse) {
return false;
@ -94,7 +93,7 @@ public class Booleans {
return true;
}
throw new IllegalArgumentException("value cannot be parsed to boolean [ true/1/on/yes OR false/0/off/no ] ");
throw new IllegalArgumentException("Failed to parse value [" + value + "] cannot be parsed to boolean [ true/1/on/yes OR false/0/off/no ]");
}
public static Boolean parseBoolean(String value, Boolean defaultValue) {

120
core/src/main/java/org/elasticsearch/common/Randomness.java Normal file
View File

@ -0,0 +1,120 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common;
import org.elasticsearch.common.settings.Settings;
import java.lang.reflect.Method;
import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
/**
* Provides factory methods for producing reproducible sources of
* randomness. Reproducible sources of randomness contribute to
* reproducible tests. When running the Elasticsearch test suite, the
* test runner will establish a global random seed accessible via the
* system property "tests.seed". By seeding a random number generator
* with this global seed, we ensure that instances of Random produced
* with this class produce reproducible sources of randomness under
* when running under the Elasticsearch test suite. Alternatively,
* a reproducible source of randomness can be produced by providing a
* setting a reproducible seed. When running the Elasticsearch server
* process, non-reproducible sources of randomness are provided (unless
* a setting is provided for a module that exposes a seed setting (e.g.,
* DiscoveryService#SETTING_DISCOVERY_SEED)).
*/
public final class Randomness {
private static final Method currentMethod;
private static final Method getRandomMethod;
static {
Method maybeCurrentMethod;
Method maybeGetRandomMethod;
try {
Class<?> clazz = Class.forName("com.carrotsearch.randomizedtesting.RandomizedContext");
maybeCurrentMethod = clazz.getMethod("current");
maybeGetRandomMethod = clazz.getMethod("getRandom");
} catch (Throwable t) {
maybeCurrentMethod = null;
maybeGetRandomMethod = null;
}
currentMethod = maybeCurrentMethod;
getRandomMethod = maybeGetRandomMethod;
}
private Randomness() {}
/**
* Provides a reproducible source of randomness seeded by a long
* seed in the settings with the key setting.
*
* @param settings the settings containing the seed
* @param setting the key to access the seed
* @return a reproducible source of randomness
*/
public static Random get(Settings settings, String setting) {
Long maybeSeed = settings.getAsLong(setting, null);
if (maybeSeed != null) {
return new Random(maybeSeed);
} else {
return get();
}
}
/**
* Provides a source of randomness that is reproducible when
* running under the Elasticsearch test suite, and otherwise
* produces a non-reproducible source of randomness. Reproducible
* sources of randomness are created when the system property
* "tests.seed" is set and the security policy allows reading this
* system property. Otherwise, non-reproducible sources of
* randomness are created.
*
* @return a source of randomness
* @throws IllegalStateException if running tests but was not able
* to acquire an instance of Random from
* RandomizedContext or tests are
* running but tests.seed is not set
*/
public static Random get() {
if (currentMethod != null && getRandomMethod != null) {
try {
Object randomizedContext = currentMethod.invoke(null);
return (Random) getRandomMethod.invoke(randomizedContext);
} catch (ReflectiveOperationException e) {
// unexpected, bail
throw new IllegalStateException("running tests but failed to invoke RandomizedContext#getRandom", e);
}
} else {
return getWithoutSeed();
}
}
private static Random getWithoutSeed() {
assert currentMethod == null && getRandomMethod == null : "running under tests but tried to create non-reproducible random";
return ThreadLocalRandom.current();
}
public static void shuffle(List<?> list) {
Collections.shuffle(list, get());
}
}

33
core/src/main/java/org/elasticsearch/common/geo/builders/EnvelopeBuilder.java
View File

@ -25,26 +25,17 @@ import com.vividsolutions.jts.geom.Coordinate;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.Locale;
import java.util.Objects;
public class EnvelopeBuilder extends ShapeBuilder {
public static final GeoShapeType TYPE = GeoShapeType.ENVELOPE;
public static final EnvelopeBuilder PROTOTYPE = new EnvelopeBuilder();
protected Coordinate topLeft;
protected Coordinate bottomRight;
public EnvelopeBuilder() {
this(Orientation.RIGHT);
}
public EnvelopeBuilder(Orientation orientation) {
super(orientation);
}
private Coordinate topLeft;
private Coordinate bottomRight;
public EnvelopeBuilder topLeft(Coordinate topLeft) {
this.topLeft = topLeft;
@ -55,6 +46,10 @@ public class EnvelopeBuilder extends ShapeBuilder {
return topLeft(coordinate(longitude, latitude));
}
public Coordinate topLeft() {
return this.topLeft;
}
public EnvelopeBuilder bottomRight(Coordinate bottomRight) {
this.bottomRight = bottomRight;
return this;
@ -64,11 +59,14 @@ public class EnvelopeBuilder extends ShapeBuilder {
return bottomRight(coordinate(longitude, latitude));
}
public Coordinate bottomRight() {
return this.bottomRight;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(FIELD_TYPE, TYPE.shapeName());
builder.field(FIELD_ORIENTATION, orientation.name().toLowerCase(Locale.ROOT));
builder.startArray(FIELD_COORDINATES);
toXContent(builder, topLeft);
toXContent(builder, bottomRight);
@ -88,7 +86,7 @@ public class EnvelopeBuilder extends ShapeBuilder {
@Override
public int hashCode() {
return Objects.hash(orientation, topLeft, bottomRight);
return Objects.hash(topLeft, bottomRight);
}
@Override
@ -100,22 +98,19 @@ public class EnvelopeBuilder extends ShapeBuilder {
return false;
}
EnvelopeBuilder other = (EnvelopeBuilder) obj;
return Objects.equals(orientation, other.orientation) &&
Objects.equals(topLeft, other.topLeft) &&
return Objects.equals(topLeft, other.topLeft) &&
Objects.equals(bottomRight, other.bottomRight);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeBoolean(orientation == Orientation.RIGHT);
writeCoordinateTo(topLeft, out);
writeCoordinateTo(bottomRight, out);
}
@Override
public EnvelopeBuilder readFrom(StreamInput in) throws IOException {
Orientation orientation = in.readBoolean() ? Orientation.RIGHT : Orientation.LEFT;
return new EnvelopeBuilder(orientation)
return new EnvelopeBuilder()
.topLeft(readCoordinateFrom(in))
.bottomRight(readCoordinateFrom(in));
}

49
core/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java
View File

@ -20,28 +20,26 @@
package org.elasticsearch.common.geo.builders;
import com.spatial4j.core.shape.Shape;
import org.elasticsearch.common.geo.XShapeCollection;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
public class GeometryCollectionBuilder extends ShapeBuilder {
public static final GeoShapeType TYPE = GeoShapeType.GEOMETRYCOLLECTION;
public static final GeometryCollectionBuilder PROTOTYPE = new GeometryCollectionBuilder();
protected final ArrayList<ShapeBuilder> shapes = new ArrayList<>();
public GeometryCollectionBuilder() {
this(Orientation.RIGHT);
}
public GeometryCollectionBuilder(Orientation orientation) {
super(orientation);
}
public GeometryCollectionBuilder shape(ShapeBuilder shape) {
this.shapes.add(shape);
return this;
@ -132,4 +130,39 @@ public class GeometryCollectionBuilder extends ShapeBuilder {
//note: ShapeCollection is probably faster than a Multi* geom.
}
@Override
public int hashCode() {
return Objects.hash(shapes);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
GeometryCollectionBuilder other = (GeometryCollectionBuilder) obj;
return Objects.equals(shapes, other.shapes);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(shapes.size());
for (ShapeBuilder shape : shapes) {
out.writeShape(shape);
}
}
@Override
public GeometryCollectionBuilder readFrom(StreamInput in) throws IOException {
GeometryCollectionBuilder geometryCollectionBuilder = new GeometryCollectionBuilder();
int shapes = in.readVInt();
for (int i = 0; i < shapes; i++) {
geometryCollectionBuilder.shape(in.readShape());
}
return geometryCollectionBuilder;
}
}

41
core/src/main/java/org/elasticsearch/common/geo/builders/LineStringBuilder.java
View File

@ -22,8 +22,12 @@ package org.elasticsearch.common.geo.builders;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Objects;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import com.spatial4j.core.shape.Shape;
import com.vividsolutions.jts.geom.Coordinate;
import com.vividsolutions.jts.geom.Geometry;
@ -34,6 +38,8 @@ public class LineStringBuilder extends PointCollection<LineStringBuilder> {
public static final GeoShapeType TYPE = GeoShapeType.LINESTRING;
public static final LineStringBuilder PROTOTYPE = new LineStringBuilder();
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
@ -139,4 +145,39 @@ public class LineStringBuilder extends PointCollection<LineStringBuilder> {
}
return coordinates;
}
@Override
public int hashCode() {
return Objects.hash(points);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
LineStringBuilder other = (LineStringBuilder) obj;
return Objects.equals(points, other.points);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(points.size());
for (Coordinate point : points) {
writeCoordinateTo(point, out);
}
}
@Override
public LineStringBuilder readFrom(StreamInput in) throws IOException {
LineStringBuilder lineStringBuilder = new LineStringBuilder();
int size = in.readVInt();
for (int i=0; i < size; i++) {
lineStringBuilder.point(readCoordinateFrom(in));
}
return lineStringBuilder;
}
}

44
core/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java
View File

@ -19,6 +19,8 @@
package org.elasticsearch.common.geo.builders;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import com.spatial4j.core.shape.Shape;
@ -29,11 +31,14 @@ import com.vividsolutions.jts.geom.LineString;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Objects;
public class MultiLineStringBuilder extends ShapeBuilder {
public static final GeoShapeType TYPE = GeoShapeType.MULTILINESTRING;
public static final MultiLineStringBuilder PROTOTYPE = new MultiLineStringBuilder();
private final ArrayList<LineStringBuilder> lines = new ArrayList<>();
public MultiLineStringBuilder linestring(LineStringBuilder line) {
@ -41,6 +46,10 @@ public class MultiLineStringBuilder extends ShapeBuilder {
return this;
}
public MultiLineStringBuilder linestring(Coordinate[] coordinates) {
return this.linestring(new LineStringBuilder().points(coordinates));
}
public Coordinate[][] coordinates() {
Coordinate[][] result = new Coordinate[lines.size()][];
for (int i = 0; i < result.length; i++) {
@ -92,4 +101,39 @@ public class MultiLineStringBuilder extends ShapeBuilder {
}
return jtsGeometry(geometry);
}
@Override
public int hashCode() {
return Objects.hash(lines);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
MultiLineStringBuilder other = (MultiLineStringBuilder) obj;
return Objects.equals(lines, other.lines);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(lines.size());
for (LineStringBuilder line : lines) {
line.writeTo(out);
}
}
@Override
public MultiLineStringBuilder readFrom(StreamInput in) throws IOException {
MultiLineStringBuilder multiLineStringBuilder = new MultiLineStringBuilder();
int size = in.readVInt();
for (int i = 0; i < size; i++) {
multiLineStringBuilder.linestring(LineStringBuilder.PROTOTYPE.readFrom(in));
}
return multiLineStringBuilder;
}
}

47
core/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java
View File

@ -22,18 +22,22 @@ package org.elasticsearch.common.geo.builders;
import com.spatial4j.core.shape.Point;
import com.spatial4j.core.shape.Shape;
import com.vividsolutions.jts.geom.Coordinate;
import org.elasticsearch.common.geo.XShapeCollection;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.geo.XShapeCollection;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
public class MultiPointBuilder extends PointCollection<MultiPointBuilder> {
public static final GeoShapeType TYPE = GeoShapeType.MULTIPOINT;
public final static MultiPointBuilder PROTOTYPE = new MultiPointBuilder();
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
@ -52,7 +56,7 @@ public class MultiPointBuilder extends PointCollection<MultiPointBuilder> {
for (Coordinate coord : points) {
shapes.add(SPATIAL_CONTEXT.makePoint(coord.x, coord.y));
}
XShapeCollection multiPoints = new XShapeCollection<>(shapes, SPATIAL_CONTEXT);
XShapeCollection<Point> multiPoints = new XShapeCollection<>(shapes, SPATIAL_CONTEXT);
multiPoints.setPointsOnly(true);
return multiPoints;
}
@ -61,4 +65,39 @@ public class MultiPointBuilder extends PointCollection<MultiPointBuilder> {
public GeoShapeType type() {
return TYPE;
}
@Override
public int hashCode() {
return Objects.hash(points);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
MultiPointBuilder other = (MultiPointBuilder) obj;
return Objects.equals(points, other.points);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(points.size());
for (Coordinate point : points) {
writeCoordinateTo(point, out);
}
}
@Override
public MultiPointBuilder readFrom(StreamInput in) throws IOException {
MultiPointBuilder multiPointBuilder = new MultiPointBuilder();
int size = in.readVInt();
for (int i=0; i < size; i++) {
multiPointBuilder.point(readCoordinateFrom(in));
}
return multiPointBuilder;
}
}

71
core/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java
View File

@ -22,8 +22,12 @@ package org.elasticsearch.common.geo.builders;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import org.elasticsearch.common.geo.XShapeCollection;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import com.spatial4j.core.shape.Shape;
@ -32,26 +36,50 @@ import com.vividsolutions.jts.geom.Coordinate;
public class MultiPolygonBuilder extends ShapeBuilder {
public static final GeoShapeType TYPE = GeoShapeType.MULTIPOLYGON;
public static final MultiPolygonBuilder PROTOTYPE = new MultiPolygonBuilder();
protected final ArrayList<PolygonBuilder> polygons = new ArrayList<>();
private final ArrayList<PolygonBuilder> polygons = new ArrayList<>();
private Orientation orientation = Orientation.RIGHT;
public MultiPolygonBuilder() {
this(Orientation.RIGHT);
}
public MultiPolygonBuilder(Orientation orientation) {
super(orientation);
this.orientation = orientation;
}
public Orientation orientation() {
return this.orientation;
}
/**
* Add a shallow copy of the polygon to the multipolygon. This will apply the orientation of the
* {@link MultiPolygonBuilder} to the polygon if polygon has different orientation.
*/
public MultiPolygonBuilder polygon(PolygonBuilder polygon) {
this.polygons.add(polygon);
PolygonBuilder pb = new PolygonBuilder(this.orientation);
pb.points(polygon.shell().coordinates(false));
for (LineStringBuilder hole : polygon.holes()) {
pb.hole(hole);
}
this.polygons.add(pb);
return this;
}
/**
* get the list of polygons
*/
public ArrayList<PolygonBuilder> polygons() {
return polygons;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(FIELD_TYPE, TYPE.shapeName());
builder.field(FIELD_ORIENTATION, orientation.name().toLowerCase(Locale.ROOT));
builder.startArray(FIELD_COORDINATES);
for(PolygonBuilder polygon : polygons) {
builder.startArray();
@ -89,4 +117,41 @@ public class MultiPolygonBuilder extends ShapeBuilder {
return new XShapeCollection<>(shapes, SPATIAL_CONTEXT);
//note: ShapeCollection is probably faster than a Multi* geom.
}
@Override
public int hashCode() {
return Objects.hash(polygons, orientation);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
MultiPolygonBuilder other = (MultiPolygonBuilder) obj;
return Objects.equals(polygons, other.polygons) &&
Objects.equals(orientation, other.orientation);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
orientation.writeTo(out);
out.writeVInt(polygons.size());
for (PolygonBuilder polygon : polygons) {
polygon.writeTo(out);
}
}
@Override
public MultiPolygonBuilder readFrom(StreamInput in) throws IOException {
MultiPolygonBuilder polyBuilder = new MultiPolygonBuilder(Orientation.readFrom(in));
int holes = in.readVInt();
for (int i = 0; i < holes; i++) {
polyBuilder.polygon(PolygonBuilder.PROTOTYPE.readFrom(in));
}
return polyBuilder;
}
}

1
core/src/main/java/org/elasticsearch/common/geo/builders/PointBuilder.java
View File

@ -32,7 +32,6 @@ import com.vividsolutions.jts.geom.Coordinate;
public class PointBuilder extends ShapeBuilder {
public static final GeoShapeType TYPE = GeoShapeType.POINT;
public static final PointBuilder PROTOTYPE = new PointBuilder();
private Coordinate coordinate;

75
core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
View File

@ -29,6 +29,8 @@ import com.vividsolutions.jts.geom.MultiPolygon;
import com.vividsolutions.jts.geom.Polygon;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.XContentBuilder;
@ -39,6 +41,9 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
/**
* The {@link PolygonBuilder} implements the groundwork to create polygons. This contains
@ -48,6 +53,11 @@ import java.util.concurrent.atomic.AtomicBoolean;
public class PolygonBuilder extends ShapeBuilder {
public static final GeoShapeType TYPE = GeoShapeType.POLYGON;
public static final PolygonBuilder PROTOTYPE = new PolygonBuilder();
private static final Coordinate[][] EMPTY = new Coordinate[0][];
private Orientation orientation = Orientation.RIGHT;
// line string defining the shell of the polygon
private LineStringBuilder shell;
@ -56,7 +66,7 @@ public class PolygonBuilder extends ShapeBuilder {
private final ArrayList<LineStringBuilder> holes = new ArrayList<>();
public PolygonBuilder() {
this(new ArrayList<Coordinate>(), Orientation.RIGHT);
this(Orientation.RIGHT);
}
public PolygonBuilder(Orientation orientation) {
@ -64,10 +74,14 @@ public class PolygonBuilder extends ShapeBuilder {
}
public PolygonBuilder(ArrayList<Coordinate> points, Orientation orientation) {
super(orientation);
this.orientation = orientation;
this.shell = new LineStringBuilder().points(points);
}
public Orientation orientation() {
return this.orientation;
}
public PolygonBuilder point(double longitude, double latitude) {
shell.point(longitude, latitude);
return this;
@ -103,6 +117,20 @@ public class PolygonBuilder extends ShapeBuilder {
return this;
}
/**
* @return the list of holes defined for this polygon
*/
public List<LineStringBuilder> holes() {
return this.holes;
}
/**
* @return the list of points of the shell for this polygon
*/
public LineStringBuilder shell() {
return this.shell;
}
/**
* Close the shell of the polygon
*/
@ -175,6 +203,7 @@ public class PolygonBuilder extends ShapeBuilder {
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(FIELD_TYPE, TYPE.shapeName());
builder.field(FIELD_ORIENTATION, orientation.name().toLowerCase(Locale.ROOT));
builder.startArray(FIELD_COORDINATES);
coordinatesArray(builder, params);
builder.endArray();
@ -357,8 +386,6 @@ public class PolygonBuilder extends ShapeBuilder {
return result;
}
private static final Coordinate[][] EMPTY = new Coordinate[0][];
private static Coordinate[][] holes(Edge[] holes, int numHoles) {
if (numHoles == 0) {
return EMPTY;
@ -663,4 +690,44 @@ public class PolygonBuilder extends ShapeBuilder {
}
}
}
@Override
public int hashCode() {
return Objects.hash(shell, holes, orientation);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
PolygonBuilder other = (PolygonBuilder) obj;
return Objects.equals(shell, other.shell) &&
Objects.equals(holes, other.holes) &&
Objects.equals(orientation, other.orientation);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
orientation.writeTo(out);
shell.writeTo(out);
out.writeVInt(holes.size());
for (LineStringBuilder hole : holes) {
hole.writeTo(out);
}
}
@Override
public PolygonBuilder readFrom(StreamInput in) throws IOException {
PolygonBuilder polyBuilder = new PolygonBuilder(Orientation.readFrom(in));
polyBuilder.shell = LineStringBuilder.PROTOTYPE.readFrom(in);
int holes = in.readVInt();
for (int i = 0; i < holes; i++) {
polyBuilder.hole(LineStringBuilder.PROTOTYPE.readFrom(in));
}
return polyBuilder;
}
}

68
core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java
View File

@ -77,16 +77,10 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
/** @see com.spatial4j.core.shape.jts.JtsGeometry#index() */
protected final boolean autoIndexJtsGeometry = true;//may want to turn off once SpatialStrategy impls do it.
protected Orientation orientation = Orientation.RIGHT;
protected ShapeBuilder() {
}
protected ShapeBuilder(Orientation orientation) {
this.orientation = orientation;
}
protected static Coordinate coordinate(double longitude, double latitude) {
return new Coordinate(longitude, latitude);
}
@ -186,22 +180,6 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
return new Coordinate(in.readDouble(), in.readDouble());
}
public static Orientation orientationFromString(String orientation) {
orientation = orientation.toLowerCase(Locale.ROOT);
switch (orientation) {
case "right":
case "counterclockwise":
case "ccw":
return Orientation.RIGHT;
case "left":
case "clockwise":
case "cw":
return Orientation.LEFT;
default:
throw new IllegalArgumentException("Unknown orientation [" + orientation + "]");
}
}
protected static Coordinate shift(Coordinate coordinate, double dateline) {
if (dateline == 0) {
return coordinate;
@ -408,6 +386,30 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
public static final Orientation COUNTER_CLOCKWISE = Orientation.RIGHT;
public static final Orientation CW = Orientation.LEFT;
public static final Orientation CCW = Orientation.RIGHT;
public void writeTo (StreamOutput out) throws IOException {
out.writeBoolean(this == Orientation.RIGHT);
}
public static Orientation readFrom (StreamInput in) throws IOException {
return in.readBoolean() ? Orientation.RIGHT : Orientation.LEFT;
}
public static Orientation fromString(String orientation) {
orientation = orientation.toLowerCase(Locale.ROOT);
switch (orientation) {
case "right":
case "counterclockwise":
case "ccw":
return Orientation.RIGHT;
case "left":
case "clockwise":
case "cw":
return Orientation.LEFT;
default:
throw new IllegalArgumentException("Unknown orientation [" + orientation + "]");
}
}
}
public static final String FIELD_TYPE = "type";
@ -498,7 +500,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
radius = Distance.parseDistance(parser.text());
} else if (FIELD_ORIENTATION.equals(fieldName)) {
parser.nextToken();
requestedOrientation = orientationFromString(parser.text());
requestedOrientation = Orientation.fromString(parser.text());
} else {
parser.nextToken();
parser.skipChildren();
@ -524,7 +526,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
case POLYGON: return parsePolygon(node, requestedOrientation, coerce);
case MULTIPOLYGON: return parseMultiPolygon(node, requestedOrientation, coerce);
case CIRCLE: return parseCircle(node, radius);
case ENVELOPE: return parseEnvelope(node, requestedOrientation);
case ENVELOPE: return parseEnvelope(node);
case GEOMETRYCOLLECTION: return geometryCollections;
default:
throw new ElasticsearchParseException("shape type [{}] not included", shapeType);
@ -550,7 +552,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
return ShapeBuilders.newCircleBuilder().center(coordinates.coordinate).radius(radius);
}
protected static EnvelopeBuilder parseEnvelope(CoordinateNode coordinates, final Orientation orientation) {
protected static EnvelopeBuilder parseEnvelope(CoordinateNode coordinates) {
// validate the coordinate array for envelope type
if (coordinates.children.size() != 2) {
throw new ElasticsearchParseException("invalid number of points [{}] provided for " +
@ -564,7 +566,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
uL = new Coordinate(Math.min(uL.x, lR.x), Math.max(uL.y, lR.y));
lR = new Coordinate(Math.max(uLtmp.x, lR.x), Math.min(uLtmp.y, lR.y));
}
return ShapeBuilders.newEnvelope(orientation).topLeft(uL).bottomRight(lR);
return ShapeBuilders.newEnvelope().topLeft(uL).bottomRight(lR);
}
protected static void validateMultiPointNode(CoordinateNode coordinates) {
@ -684,8 +686,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
}
XContentParser.Token token = parser.nextToken();
GeometryCollectionBuilder geometryCollection = ShapeBuilders.newGeometryCollection( (mapper == null) ? Orientation.RIGHT : mapper
.fieldType().orientation());
GeometryCollectionBuilder geometryCollection = ShapeBuilders.newGeometryCollection();
while (token != XContentParser.Token.END_ARRAY) {
ShapeBuilder shapeBuilder = GeoShapeType.parse(parser);
geometryCollection.shape(shapeBuilder);
@ -700,15 +701,4 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
public String getWriteableName() {
return type().shapeName();
}
// NORELEASE this should be deleted as soon as all shape builders implement writable
@Override
public void writeTo(StreamOutput out) throws IOException {
}
// NORELEASE this should be deleted as soon as all shape builders implement writable
@Override
public ShapeBuilder readFrom(StreamInput in) throws IOException {
return null;
}
}

45
core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilderRegistry.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.geo.builders;
import org.elasticsearch.common.geo.ShapesAvailability;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
/**
* Register the shape builder prototypes with the {@link NamedWriteableRegistry}
*/
public class ShapeBuilderRegistry {
@Inject
public ShapeBuilderRegistry(NamedWriteableRegistry namedWriteableRegistry) {
if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) {
namedWriteableRegistry.registerPrototype(ShapeBuilder.class, PointBuilder.PROTOTYPE);
namedWriteableRegistry.registerPrototype(ShapeBuilder.class, CircleBuilder.PROTOTYPE);
namedWriteableRegistry.registerPrototype(ShapeBuilder.class, EnvelopeBuilder.PROTOTYPE);
namedWriteableRegistry.registerPrototype(ShapeBuilder.class, MultiPointBuilder.PROTOTYPE);
namedWriteableRegistry.registerPrototype(ShapeBuilder.class, LineStringBuilder.PROTOTYPE);
namedWriteableRegistry.registerPrototype(ShapeBuilder.class, MultiLineStringBuilder.PROTOTYPE);
namedWriteableRegistry.registerPrototype(ShapeBuilder.class, PolygonBuilder.PROTOTYPE);
namedWriteableRegistry.registerPrototype(ShapeBuilder.class, MultiPolygonBuilder.PROTOTYPE);
namedWriteableRegistry.registerPrototype(ShapeBuilder.class, GeometryCollectionBuilder.PROTOTYPE);
}
}
}

18
core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilders.java
View File

@ -110,15 +110,6 @@ public class ShapeBuilders {
return new GeometryCollectionBuilder();
}
/**
* Create a new GeometryCollection
*
* @return a new {@link GeometryCollectionBuilder}
*/
public static GeometryCollectionBuilder newGeometryCollection(ShapeBuilder.Orientation orientation) {
return new GeometryCollectionBuilder(orientation);
}
/**
* create a new Circle
*
@ -136,13 +127,4 @@ public class ShapeBuilders {
public static EnvelopeBuilder newEnvelope() {
return new EnvelopeBuilder();
}
/**
* create a new rectangle
*
* @return a new {@link EnvelopeBuilder}
*/
public static EnvelopeBuilder newEnvelope(ShapeBuilder.Orientation orientation) {
return new EnvelopeBuilder(orientation);
}
}

13
core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
View File

@ -33,7 +33,7 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.text.StringAndBytesText;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
@ -256,13 +256,13 @@ public abstract class StreamInput extends InputStream {
if (length == -1) {
return null;
}
return new StringAndBytesText(readBytesReference(length));
return new Text(readBytesReference(length));
}
public Text readText() throws IOException {
// use StringAndBytes so we can cache the string if its ever converted to it
int length = readInt();
return new StringAndBytesText(readBytesReference(length));
return new Text(readBytesReference(length));
}
@Nullable
@ -629,6 +629,13 @@ public abstract class StreamInput extends InputStream {
return readNamedWriteable(QueryBuilder.class);
}
/**
* Reads a {@link ShapeBuilder} from the current stream
*/
public ShapeBuilder readShape() throws IOException {
return readNamedWriteable(ShapeBuilder.class);
}
/**
* Reads a {@link org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder} from the current stream
*/

8
core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
View File

@ -32,6 +32,7 @@ import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
@ -618,6 +619,13 @@ public abstract class StreamOutput extends OutputStream {
writeNamedWriteable(queryBuilder);
}
/**
* Writes a {@link ShapeBuilder} to the current stream
*/
public void writeShape(ShapeBuilder shapeBuilder) throws IOException {
writeNamedWriteable(shapeBuilder);
}
/**
* Writes a {@link ScoreFunctionBuilder} to the current stream
*/

4
core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java
View File

@ -149,6 +149,10 @@ public final class AllTermQuery extends Query {
return null;
}
final TermState state = termStates.get(context.ord);
if (state == null) {
// Term does not exist in this segment
return null;
}
termsEnum.seekExact(term.bytes(), state);
PostingsEnum docs = termsEnum.postings(null, PostingsEnum.PAYLOADS);
assert docs != null;

347
core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
View File

@ -19,21 +19,362 @@
package org.elasticsearch.common.network;
import org.elasticsearch.client.support.Headers;
import org.elasticsearch.client.transport.TransportClientNodesService;
import org.elasticsearch.client.transport.support.TransportProxyClient;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.ExtensionPoint;
import org.elasticsearch.http.HttpServer;
import org.elasticsearch.http.HttpServerTransport;
import org.elasticsearch.http.netty.NettyHttpServerTransport;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.rest.action.admin.cluster.health.RestClusterHealthAction;
import org.elasticsearch.rest.action.admin.cluster.node.hotthreads.RestNodesHotThreadsAction;
import org.elasticsearch.rest.action.admin.cluster.node.info.RestNodesInfoAction;
import org.elasticsearch.rest.action.admin.cluster.node.stats.RestNodesStatsAction;
import org.elasticsearch.rest.action.admin.cluster.repositories.delete.RestDeleteRepositoryAction;
import org.elasticsearch.rest.action.admin.cluster.repositories.get.RestGetRepositoriesAction;
import org.elasticsearch.rest.action.admin.cluster.repositories.put.RestPutRepositoryAction;
import org.elasticsearch.rest.action.admin.cluster.repositories.verify.RestVerifyRepositoryAction;
import org.elasticsearch.rest.action.admin.cluster.reroute.RestClusterRerouteAction;
import org.elasticsearch.rest.action.admin.cluster.settings.RestClusterGetSettingsAction;
import org.elasticsearch.rest.action.admin.cluster.settings.RestClusterUpdateSettingsAction;
import org.elasticsearch.rest.action.admin.cluster.shards.RestClusterSearchShardsAction;
import org.elasticsearch.rest.action.admin.cluster.snapshots.create.RestCreateSnapshotAction;
import org.elasticsearch.rest.action.admin.cluster.snapshots.delete.RestDeleteSnapshotAction;
import org.elasticsearch.rest.action.admin.cluster.snapshots.get.RestGetSnapshotsAction;
import org.elasticsearch.rest.action.admin.cluster.snapshots.restore.RestRestoreSnapshotAction;
import org.elasticsearch.rest.action.admin.cluster.snapshots.status.RestSnapshotsStatusAction;
import org.elasticsearch.rest.action.admin.cluster.state.RestClusterStateAction;
import org.elasticsearch.rest.action.admin.cluster.stats.RestClusterStatsAction;
import org.elasticsearch.rest.action.admin.cluster.tasks.RestPendingClusterTasksAction;
import org.elasticsearch.rest.action.admin.indices.alias.RestIndicesAliasesAction;
import org.elasticsearch.rest.action.admin.indices.alias.delete.RestIndexDeleteAliasesAction;
import org.elasticsearch.rest.action.admin.indices.alias.get.RestGetAliasesAction;
import org.elasticsearch.rest.action.admin.indices.alias.get.RestGetIndicesAliasesAction;
import org.elasticsearch.rest.action.admin.indices.alias.head.RestAliasesExistAction;
import org.elasticsearch.rest.action.admin.indices.alias.put.RestIndexPutAliasAction;
import org.elasticsearch.rest.action.admin.indices.analyze.RestAnalyzeAction;
import org.elasticsearch.rest.action.admin.indices.cache.clear.RestClearIndicesCacheAction;
import org.elasticsearch.rest.action.admin.indices.close.RestCloseIndexAction;
import org.elasticsearch.rest.action.admin.indices.create.RestCreateIndexAction;
import org.elasticsearch.rest.action.admin.indices.delete.RestDeleteIndexAction;
import org.elasticsearch.rest.action.admin.indices.exists.indices.RestIndicesExistsAction;
import org.elasticsearch.rest.action.admin.indices.exists.types.RestTypesExistsAction;
import org.elasticsearch.rest.action.admin.indices.flush.RestFlushAction;
import org.elasticsearch.rest.action.admin.indices.flush.RestSyncedFlushAction;
import org.elasticsearch.rest.action.admin.indices.forcemerge.RestForceMergeAction;
import org.elasticsearch.rest.action.admin.indices.get.RestGetIndicesAction;
import org.elasticsearch.rest.action.admin.indices.mapping.get.RestGetFieldMappingAction;
import org.elasticsearch.rest.action.admin.indices.mapping.get.RestGetMappingAction;
import org.elasticsearch.rest.action.admin.indices.mapping.put.RestPutMappingAction;
import org.elasticsearch.rest.action.admin.indices.open.RestOpenIndexAction;
import org.elasticsearch.rest.action.admin.indices.recovery.RestRecoveryAction;
import org.elasticsearch.rest.action.admin.indices.refresh.RestRefreshAction;
import org.elasticsearch.rest.action.admin.indices.segments.RestIndicesSegmentsAction;
import org.elasticsearch.rest.action.admin.indices.settings.RestGetSettingsAction;
import org.elasticsearch.rest.action.admin.indices.settings.RestUpdateSettingsAction;
import org.elasticsearch.rest.action.admin.indices.shards.RestIndicesShardStoresAction;
import org.elasticsearch.rest.action.admin.indices.stats.RestIndicesStatsAction;
import org.elasticsearch.rest.action.admin.indices.template.delete.RestDeleteIndexTemplateAction;
import org.elasticsearch.rest.action.admin.indices.template.get.RestGetIndexTemplateAction;
import org.elasticsearch.rest.action.admin.indices.template.head.RestHeadIndexTemplateAction;
import org.elasticsearch.rest.action.admin.indices.template.put.RestPutIndexTemplateAction;
import org.elasticsearch.rest.action.admin.indices.upgrade.RestUpgradeAction;
import org.elasticsearch.rest.action.admin.indices.validate.query.RestValidateQueryAction;
import org.elasticsearch.rest.action.admin.indices.validate.template.RestRenderSearchTemplateAction;
import org.elasticsearch.rest.action.admin.indices.warmer.delete.RestDeleteWarmerAction;
import org.elasticsearch.rest.action.admin.indices.warmer.get.RestGetWarmerAction;
import org.elasticsearch.rest.action.admin.indices.warmer.put.RestPutWarmerAction;
import org.elasticsearch.rest.action.bulk.RestBulkAction;
import org.elasticsearch.rest.action.cat.AbstractCatAction;
import org.elasticsearch.rest.action.cat.RestAliasAction;
import org.elasticsearch.rest.action.cat.RestAllocationAction;
import org.elasticsearch.rest.action.cat.RestCatAction;
import org.elasticsearch.rest.action.cat.RestFielddataAction;
import org.elasticsearch.rest.action.cat.RestHealthAction;
import org.elasticsearch.rest.action.cat.RestIndicesAction;
import org.elasticsearch.rest.action.cat.RestMasterAction;
import org.elasticsearch.rest.action.cat.RestNodeAttrsAction;
import org.elasticsearch.rest.action.cat.RestNodesAction;
import org.elasticsearch.rest.action.cat.RestPluginsAction;
import org.elasticsearch.rest.action.cat.RestRepositoriesAction;
import org.elasticsearch.rest.action.cat.RestSegmentsAction;
import org.elasticsearch.rest.action.cat.RestShardsAction;
import org.elasticsearch.rest.action.cat.RestSnapshotAction;
import org.elasticsearch.rest.action.cat.RestThreadPoolAction;
import org.elasticsearch.rest.action.delete.RestDeleteAction;
import org.elasticsearch.rest.action.explain.RestExplainAction;
import org.elasticsearch.rest.action.fieldstats.RestFieldStatsAction;
import org.elasticsearch.rest.action.get.RestGetAction;
import org.elasticsearch.rest.action.get.RestGetSourceAction;
import org.elasticsearch.rest.action.get.RestHeadAction;
import org.elasticsearch.rest.action.get.RestMultiGetAction;
import org.elasticsearch.rest.action.index.RestIndexAction;
import org.elasticsearch.rest.action.main.RestMainAction;
import org.elasticsearch.rest.action.percolate.RestMultiPercolateAction;
import org.elasticsearch.rest.action.percolate.RestPercolateAction;
import org.elasticsearch.rest.action.script.RestDeleteIndexedScriptAction;
import org.elasticsearch.rest.action.script.RestGetIndexedScriptAction;
import org.elasticsearch.rest.action.script.RestPutIndexedScriptAction;
import org.elasticsearch.rest.action.search.RestClearScrollAction;
import org.elasticsearch.rest.action.search.RestMultiSearchAction;
import org.elasticsearch.rest.action.search.RestSearchAction;
import org.elasticsearch.rest.action.search.RestSearchScrollAction;
import org.elasticsearch.rest.action.suggest.RestSuggestAction;
import org.elasticsearch.rest.action.template.RestDeleteSearchTemplateAction;
import org.elasticsearch.rest.action.template.RestGetSearchTemplateAction;
import org.elasticsearch.rest.action.template.RestPutSearchTemplateAction;
import org.elasticsearch.rest.action.termvectors.RestMultiTermVectorsAction;
import org.elasticsearch.rest.action.termvectors.RestTermVectorsAction;
import org.elasticsearch.rest.action.update.RestUpdateAction;
import org.elasticsearch.transport.Transport;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.transport.local.LocalTransport;
import org.elasticsearch.transport.netty.NettyTransport;
import java.util.Arrays;
import java.util.List;
/**
*
* A module to handle registering and binding all network related classes.
*/
public class NetworkModule extends AbstractModule {
private final NetworkService networkService;
public static final String TRANSPORT_TYPE_KEY = "transport.type";
public static final String TRANSPORT_SERVICE_TYPE_KEY = "transport.service.type";
public NetworkModule(NetworkService networkService) {
public static final String LOCAL_TRANSPORT = "local";
public static final String NETTY_TRANSPORT = "netty";
public static final String HTTP_TYPE_KEY = "http.type";
public static final String HTTP_ENABLED = "http.enabled";
private static final List<Class<? extends RestHandler>> builtinRestHandlers = Arrays.asList(
RestMainAction.class,
RestNodesInfoAction.class,
RestNodesStatsAction.class,
RestNodesHotThreadsAction.class,
RestClusterStatsAction.class,
RestClusterStateAction.class,
RestClusterHealthAction.class,
RestClusterUpdateSettingsAction.class,
RestClusterGetSettingsAction.class,
RestClusterRerouteAction.class,
RestClusterSearchShardsAction.class,
RestPendingClusterTasksAction.class,
RestPutRepositoryAction.class,
RestGetRepositoriesAction.class,
RestDeleteRepositoryAction.class,
RestVerifyRepositoryAction.class,
RestGetSnapshotsAction.class,
RestCreateSnapshotAction.class,
RestRestoreSnapshotAction.class,
RestDeleteSnapshotAction.class,
RestSnapshotsStatusAction.class,
RestIndicesExistsAction.class,
RestTypesExistsAction.class,
RestGetIndicesAction.class,
RestIndicesStatsAction.class,
RestIndicesSegmentsAction.class,
RestIndicesShardStoresAction.class,
RestGetAliasesAction.class,
RestAliasesExistAction.class,
RestIndexDeleteAliasesAction.class,
RestIndexPutAliasAction.class,
RestIndicesAliasesAction.class,
RestGetIndicesAliasesAction.class,
RestCreateIndexAction.class,
RestDeleteIndexAction.class,
RestCloseIndexAction.class,
RestOpenIndexAction.class,
RestUpdateSettingsAction.class,
RestGetSettingsAction.class,
RestAnalyzeAction.class,
RestGetIndexTemplateAction.class,
RestPutIndexTemplateAction.class,
RestDeleteIndexTemplateAction.class,
RestHeadIndexTemplateAction.class,
RestPutWarmerAction.class,
RestDeleteWarmerAction.class,
RestGetWarmerAction.class,
RestPutMappingAction.class,
RestGetMappingAction.class,
RestGetFieldMappingAction.class,
RestRefreshAction.class,
RestFlushAction.class,
RestSyncedFlushAction.class,
RestForceMergeAction.class,
RestUpgradeAction.class,
RestClearIndicesCacheAction.class,
RestIndexAction.class,
RestGetAction.class,
RestGetSourceAction.class,
RestHeadAction.class,
RestMultiGetAction.class,
RestDeleteAction.class,
org.elasticsearch.rest.action.count.RestCountAction.class,
RestSuggestAction.class,
RestTermVectorsAction.class,
RestMultiTermVectorsAction.class,
RestBulkAction.class,
RestUpdateAction.class,
RestPercolateAction.class,
RestMultiPercolateAction.class,
RestSearchAction.class,
RestSearchScrollAction.class,
RestClearScrollAction.class,
RestMultiSearchAction.class,
RestRenderSearchTemplateAction.class,
RestValidateQueryAction.class,
RestExplainAction.class,
RestRecoveryAction.class,
// Templates API
RestGetSearchTemplateAction.class,
RestPutSearchTemplateAction.class,
RestDeleteSearchTemplateAction.class,
// Scripts API
RestGetIndexedScriptAction.class,
RestPutIndexedScriptAction.class,
RestDeleteIndexedScriptAction.class,
RestFieldStatsAction.class,
// no abstract cat action
RestCatAction.class
);
private static final List<Class<? extends AbstractCatAction>> builtinCatHandlers = Arrays.asList(
RestAllocationAction.class,
RestShardsAction.class,
RestMasterAction.class,
RestNodesAction.class,
RestIndicesAction.class,
RestSegmentsAction.class,
// Fully qualified to prevent interference with rest.action.count.RestCountAction
org.elasticsearch.rest.action.cat.RestCountAction.class,
// Fully qualified to prevent interference with rest.action.indices.RestRecoveryAction
org.elasticsearch.rest.action.cat.RestRecoveryAction.class,
RestHealthAction.class,
org.elasticsearch.rest.action.cat.RestPendingClusterTasksAction.class,
RestAliasAction.class,
RestThreadPoolAction.class,
RestPluginsAction.class,
RestFielddataAction.class,
RestNodeAttrsAction.class,
RestRepositoriesAction.class,
RestSnapshotAction.class
);
private final NetworkService networkService;
private final Settings settings;
private final boolean transportClient;
private final ExtensionPoint.SelectedType<TransportService> transportServiceTypes = new ExtensionPoint.SelectedType<>("transport_service", TransportService.class);
private final ExtensionPoint.SelectedType<Transport> transportTypes = new ExtensionPoint.SelectedType<>("transport", Transport.class);
private final ExtensionPoint.SelectedType<HttpServerTransport> httpTransportTypes = new ExtensionPoint.SelectedType<>("http_transport", HttpServerTransport.class);
private final ExtensionPoint.ClassSet<RestHandler> restHandlers = new ExtensionPoint.ClassSet<>("rest_handler", RestHandler.class);
// we must separate the cat rest handlers so RestCatAction can collect them...
private final ExtensionPoint.ClassSet<AbstractCatAction> catHandlers = new ExtensionPoint.ClassSet<>("cat_handler", AbstractCatAction.class);
/**
* Creates a network module that custom networking classes can be plugged into.
*
* @param networkService A constructed network service object to bind.
* @param settings The settings for the node
* @param transportClient True if only transport classes should be allowed to be registered, false otherwise.
*/
public NetworkModule(NetworkService networkService, Settings settings, boolean transportClient) {
this.networkService = networkService;
this.settings = settings;
this.transportClient = transportClient;
registerTransportService(NETTY_TRANSPORT, TransportService.class);
registerTransport(LOCAL_TRANSPORT, LocalTransport.class);
registerTransport(NETTY_TRANSPORT, NettyTransport.class);
if (transportClient == false) {
registerHttpTransport(NETTY_TRANSPORT, NettyHttpServerTransport.class);
for (Class<? extends AbstractCatAction> catAction : builtinCatHandlers) {
catHandlers.registerExtension(catAction);
}
for (Class<? extends RestHandler> restAction : builtinRestHandlers) {
restHandlers.registerExtension(restAction);
}
}
}
/** Adds a transport service implementation that can be selected by setting {@link #TRANSPORT_SERVICE_TYPE_KEY}. */
public void registerTransportService(String name, Class<? extends TransportService> clazz) {
transportServiceTypes.registerExtension(name, clazz);
}
/** Adds a transport implementation that can be selected by setting {@link #TRANSPORT_TYPE_KEY}. */
public void registerTransport(String name, Class<? extends Transport> clazz) {
transportTypes.registerExtension(name, clazz);
}
/** Adds an http transport implementation that can be selected by setting {@link #HTTP_TYPE_KEY}. */
// TODO: we need another name than "http transport"....so confusing with transportClient...
public void registerHttpTransport(String name, Class<? extends HttpServerTransport> clazz) {
if (transportClient) {
throw new IllegalArgumentException("Cannot register http transport " + clazz.getName() + " for transport client");
}
httpTransportTypes.registerExtension(name, clazz);
}
/** Adds an additional rest action. */
// TODO: change this further to eliminate the middle man, ie RestController, and just register method and path here
public void registerRestHandler(Class<? extends RestHandler> clazz) {
if (transportClient) {
throw new IllegalArgumentException("Cannot register rest handler " + clazz.getName() + " for transport client");
}
if (AbstractCatAction.class.isAssignableFrom(clazz)) {
catHandlers.registerExtension(clazz.asSubclass(AbstractCatAction.class));
} else {
restHandlers.registerExtension(clazz);
}
}
@Override
protected void configure() {
bind(NetworkService.class).toInstance(networkService);
bind(NamedWriteableRegistry.class).asEagerSingleton();
transportServiceTypes.bindType(binder(), settings, TRANSPORT_SERVICE_TYPE_KEY, NETTY_TRANSPORT);
String defaultTransport = DiscoveryNode.localNode(settings) ? LOCAL_TRANSPORT : NETTY_TRANSPORT;
transportTypes.bindType(binder(), settings, TRANSPORT_TYPE_KEY, defaultTransport);
if (transportClient) {
bind(Headers.class).asEagerSingleton();
bind(TransportProxyClient.class).asEagerSingleton();
bind(TransportClientNodesService.class).asEagerSingleton();
} else {
if (settings.getAsBoolean(HTTP_ENABLED, true)) {
bind(HttpServer.class).asEagerSingleton();
httpTransportTypes.bindType(binder(), settings, HTTP_TYPE_KEY, NETTY_TRANSPORT);
}
bind(RestController.class).asEagerSingleton();
catHandlers.bind(binder());
restHandlers.bind(binder());
}
}
}

252
core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java Normal file
View File

@ -0,0 +1,252 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.settings;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.component.AbstractComponent;
import java.util.*;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
/**
* A basic setting service that can be used for per-index and per-cluster settings.
* This service offers transactional application of updates settings.
*/
public abstract class AbstractScopedSettings extends AbstractComponent {
private Settings lastSettingsApplied = Settings.EMPTY;
private final List<SettingUpdater> settingUpdaters = new ArrayList<>();
private final Map<String, Setting<?>> complexMatchers = new HashMap<>();
private final Map<String, Setting<?>> keySettings = new HashMap<>();
private final Setting.Scope scope;
protected AbstractScopedSettings(Settings settings, Set<Setting<?>> settingsSet, Setting.Scope scope) {
super(settings);
for (Setting<?> entry : settingsSet) {
if (entry.getScope() != scope) {
throw new IllegalArgumentException("Setting must be a cluster setting but was: " + entry.getScope());
}
if (entry.hasComplexMatcher()) {
complexMatchers.put(entry.getKey(), entry);
} else {
keySettings.put(entry.getKey(), entry);
}
}
this.scope = scope;
}
public Setting.Scope getScope() {
return this.scope;
}
/**
* Applies the given settings to all listeners and rolls back the result after application. This
* method will not change any settings but will fail if any of the settings can't be applied.
*/
public synchronized Settings dryRun(Settings settings) {
final Settings current = Settings.builder().put(this.settings).put(settings).build();
final Settings previous = Settings.builder().put(this.settings).put(this.lastSettingsApplied).build();
List<RuntimeException> exceptions = new ArrayList<>();
for (SettingUpdater settingUpdater : settingUpdaters) {
try {
if (settingUpdater.hasChanged(current, previous)) {
settingUpdater.getValue(current, previous);
}
} catch (RuntimeException ex) {
exceptions.add(ex);
logger.debug("failed to prepareCommit settings for [{}]", ex, settingUpdater);
}
}
// here we are exhaustive and record all settings that failed.
ExceptionsHelper.rethrowAndSuppress(exceptions);
return current;
}
/**
* Applies the given settings to all the settings consumers or to none of them. The settings
* will be merged with the node settings before they are applied while given settings override existing node
* settings.
* @param newSettings the settings to apply
* @return the unmerged applied settings
*/
public synchronized Settings applySettings(Settings newSettings) {
if (lastSettingsApplied != null && newSettings.equals(lastSettingsApplied)) {
// nothing changed in the settings, ignore
return newSettings;
}
final Settings current = Settings.builder().put(this.settings).put(newSettings).build();
final Settings previous = Settings.builder().put(this.settings).put(this.lastSettingsApplied).build();
try {
List<Runnable> applyRunnables = new ArrayList<>();
for (SettingUpdater settingUpdater : settingUpdaters) {
try {
applyRunnables.add(settingUpdater.updater(current, previous));
} catch (Exception ex) {
logger.warn("failed to prepareCommit settings for [{}]", ex, settingUpdater);
throw ex;
}
}
for (Runnable settingUpdater : applyRunnables) {
settingUpdater.run();
}
} catch (Exception ex) {
logger.warn("failed to apply settings", ex);
throw ex;
} finally {
}
return lastSettingsApplied = newSettings;
}
/**
* Adds a settings consumer with a predicate that is only evaluated at update time.
* <p>
* Note: Only settings registered in {@link SettingsModule} can be changed dynamically.
* </p>
* @param validator an additional validator that is only applied to updates of this setting.
* This is useful to add additional validation to settings at runtime compared to at startup time.
*/
public synchronized <T> void addSettingsUpdateConsumer(Setting<T> setting, Consumer<T> consumer, Consumer<T> validator) {
if (setting != get(setting.getKey())) {
throw new IllegalArgumentException("Setting is not registered for key [" + setting.getKey() + "]");
}
this.settingUpdaters.add(setting.newUpdater(consumer, logger, validator));
}
/**
* Adds a settings consumer that accepts the values for two settings. The consumer if only notified if one or both settings change.
* <p>
* Note: Only settings registered in {@link SettingsModule} can be changed dynamically.
* </p>
* This method registers a compound updater that is useful if two settings are depending on each other. The consumer is always provided
* with both values even if only one of the two changes.
*/
public synchronized <A, B> void addSettingsUpdateConsumer(Setting<A> a, Setting<B> b, BiConsumer<A, B> consumer) {
if (a != get(a.getKey())) {
throw new IllegalArgumentException("Setting is not registered for key [" + a.getKey() + "]");
}
if (b != get(b.getKey())) {
throw new IllegalArgumentException("Setting is not registered for key [" + b.getKey() + "]");
}
this.settingUpdaters.add(Setting.compoundUpdater(consumer, a, b, logger));
}
/**
* Adds a settings consumer.
* <p>
* Note: Only settings registered in {@link org.elasticsearch.cluster.ClusterModule} can be changed dynamically.
* </p>
*/
public synchronized <T> void addSettingsUpdateConsumer(Setting<T> setting, Consumer<T> consumer) {
addSettingsUpdateConsumer(setting, consumer, (s) -> {});
}
/**
* Transactional interface to update settings.
* @see Setting
*/
public interface SettingUpdater<T> {
/**
* Returns true if this updaters setting has changed with the current update
* @param current the current settings
* @param previous the previous setting
* @return true if this updaters setting has changed with the current update
*/
boolean hasChanged(Settings current, Settings previous);
/**
* Returns the instance value for the current settings. This method is stateless and idempotent.
* This method will throw an exception if the source of this value is invalid.
*/
T getValue(Settings current, Settings previous);
/**
* Applies the given value to the updater. This methods will actually run the update.
*/
void apply(T value, Settings current, Settings previous);
/**
* Updates this updaters value if it has changed.
* @return <code>true</code> iff the value has been updated.
*/
default boolean apply(Settings current, Settings previous) {
if (hasChanged(current, previous)) {
T value = getValue(current, previous);
apply(value, current, previous);
return true;
}
return false;
}
/**
* Returns a callable runnable that calls {@link #apply(Object, Settings, Settings)} if the settings
* actually changed. This allows to defer the update to a later point in time while keeping type safety.
* If the value didn't change the returned runnable is a noop.
*/
default Runnable updater(Settings current, Settings previous) {
if (hasChanged(current, previous)) {
T value = getValue(current, previous);
return () -> { apply(value, current, previous);};
}
return () -> {};
}
}
/**
* Returns the {@link Setting} for the given key or <code>null</code> if the setting can not be found.
*/
public Setting get(String key) {
Setting<?> setting = keySettings.get(key);
if (setting == null) {
for (Map.Entry<String, Setting<?>> entry : complexMatchers.entrySet()) {
if (entry.getValue().match(key)) {
return entry.getValue();
}
}
} else {
return setting;
}
return null;
}
/**
* Returns <code>true</code> if the setting for the given key is dynamically updateable. Otherwise <code>false</code>.
*/
public boolean hasDynamicSetting(String key) {
final Setting setting = get(key);
return setting != null && setting.isDynamic();
}
/**
* Returns a settings object that contains all settings that are not
* already set in the given source. The diff contains either the default value for each
* setting or the settings value in the given default settings.
*/
public Settings diff(Settings source, Settings defaultSettings) {
Settings.Builder builder = Settings.builder();
for (Setting<?> setting : keySettings.values()) {
if (setting.exists(source) == false) {
builder.put(setting.getKey(), setting.getRaw(defaultSettings));
}
}
return builder.build();
}
}

141
core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java Normal file
View File

@ -0,0 +1,141 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.settings;
import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction;
import org.elasticsearch.action.support.DestructiveOperations;
import org.elasticsearch.cluster.InternalClusterInfoService;
import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
import org.elasticsearch.cluster.routing.allocation.decider.*;
import org.elasticsearch.cluster.service.InternalClusterService;
import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.discovery.DiscoverySettings;
import org.elasticsearch.discovery.zen.ZenDiscovery;
import org.elasticsearch.discovery.zen.elect.ElectMasterService;
import org.elasticsearch.index.store.IndexStoreConfig;
import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
import org.elasticsearch.indices.recovery.RecoverySettings;
import org.elasticsearch.indices.ttl.IndicesTTLService;
import org.elasticsearch.search.SearchService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.Transport;
import org.elasticsearch.transport.TransportService;
import java.util.*;
/**
* Encapsulates all valid cluster level settings.
*/
public final class ClusterSettings extends AbstractScopedSettings {
public ClusterSettings(Settings settings, Set<Setting<?>> settingsSet) {
super(settings, settingsSet, Setting.Scope.CLUSTER);
}
@Override
public synchronized Settings applySettings(Settings newSettings) {
Settings settings = super.applySettings(newSettings);
try {
for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
if (entry.getKey().startsWith("logger.")) {
String component = entry.getKey().substring("logger.".length());
if ("_root".equals(component)) {
ESLoggerFactory.getRootLogger().setLevel(entry.getValue());
} else {
ESLoggerFactory.getLogger(component).setLevel(entry.getValue());
}
}
}
} catch (Exception e) {
logger.warn("failed to refresh settings for [{}]", e, "logger");
}
return settings;
}
/**
* Returns <code>true</code> if the settings is a logger setting.
*/
public boolean isLoggerSetting(String key) {
return key.startsWith("logger.");
}
public static Set<Setting<?>> BUILT_IN_CLUSTER_SETTINGS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING,
AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING,
BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING,
BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING,
BalancedShardsAllocator.THRESHOLD_SETTING,
ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING,
ConcurrentRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING,
EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING,
EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING,
ZenDiscovery.REJOIN_ON_MASTER_GONE_SETTING,
FilterAllocationDecider.CLUSTER_ROUTING_INCLUDE_GROUP_SETTING,
FilterAllocationDecider.CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING,
FilterAllocationDecider.CLUSTER_ROUTING_REQUIRE_GROUP_SETTING,
IndexStoreConfig.INDICES_STORE_THROTTLE_TYPE_SETTING,
IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING,
IndicesTTLService.INDICES_TTL_INTERVAL_SETTING,
MappingUpdatedAction.INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING,
MetaData.SETTING_READ_ONLY_SETTING,
RecoverySettings.INDICES_RECOVERY_CONCURRENT_STREAMS_SETTING,
RecoverySettings.INDICES_RECOVERY_CONCURRENT_SMALL_FILE_STREAMS_SETTING,
RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING,
RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING,
RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING,
RecoverySettings.INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING,
RecoverySettings.INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING,
RecoverySettings.INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING,
ThreadPool.THREADPOOL_GROUP_SETTING,
ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING,
ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING,
DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING,
DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING,
DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING,
DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS_SETTING,
DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING,
InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING,
InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING,
SnapshotInProgressAllocationDecider.CLUSTER_ROUTING_ALLOCATION_SNAPSHOT_RELOCATION_ENABLED_SETTING,
DestructiveOperations.REQUIRES_NAME_SETTING,
DiscoverySettings.PUBLISH_TIMEOUT_SETTING,
DiscoverySettings.PUBLISH_DIFF_ENABLE_SETTING,
DiscoverySettings.COMMIT_TIMEOUT_SETTING,
DiscoverySettings.NO_MASTER_BLOCK_SETTING,
HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING,
HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING,
HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING,
HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING,
HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING,
InternalClusterService.CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING,
SearchService.DEFAULT_SEARCH_TIMEOUT_SETTING,
ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING,
TransportService.TRACE_LOG_EXCLUDE_SETTING,
TransportService.TRACE_LOG_INCLUDE_SETTING,
TransportCloseIndexAction.CLUSTER_INDICES_CLOSE_ENABLE_SETTING,
ShardsLimitAllocationDecider.CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING,
InternalClusterService.CLUSTER_SERVICE_RECONNECT_INTERVAL_SETTING,
HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_TYPE_SETTING,
HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_TYPE_SETTING,
Transport.TRANSPORT_PROFILES_SETTING,
Transport.TRANSPORT_TCP_COMPRESS)));
}

461
core/src/main/java/org/elasticsearch/common/settings/Setting.java Normal file
View File

@ -0,0 +1,461 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.settings;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.action.support.ToXContentToBytes;
import org.elasticsearch.common.Booleans;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.MemorySizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.regex.Pattern;
/**
*/
public class Setting<T> extends ToXContentToBytes {
private final String key;
protected final Function<Settings, String> defaultValue;
private final Function<String, T> parser;
private final boolean dynamic;
private final Scope scope;
/**
* Creates a new Setting instance
* @param key the settings key for this setting.
* @param defaultValue a default value function that returns the default values string representation.
* @param parser a parser that parses the string rep into a complex datatype.
* @param dynamic true iff this setting can be dynamically updateable
* @param scope the scope of this setting
*/
public Setting(String key, Function<Settings, String> defaultValue, Function<String, T> parser, boolean dynamic, Scope scope) {
assert parser.apply(defaultValue.apply(Settings.EMPTY)) != null || this.isGroupSetting(): "parser returned null";
this.key = key;
this.defaultValue = defaultValue;
this.parser = parser;
this.dynamic = dynamic;
this.scope = scope;
}
/**
* Returns the settings key or a prefix if this setting is a group setting
* @see #isGroupSetting()
*/
public final String getKey() {
return key;
}
/**
* Returns <code>true</code> iff this setting is dynamically updateable, otherwise <code>false</code>
*/
public final boolean isDynamic() {
return dynamic;
}
/**
* Returns the settings scope
*/
public final Scope getScope() {
return scope;
}
/**
* Returns <code>true</code> iff this setting is a group setting. Group settings represent a set of settings
* rather than a single value. The key, see {@link #getKey()}, in contrast to non-group settings is a prefix like <tt>cluster.store.</tt>
* that matches all settings with this prefix.
*/
boolean isGroupSetting() {
return false;
}
boolean hasComplexMatcher() {
return isGroupSetting();
}
/**
* Returns the default values string representation for this setting.
* @param settings a settings object for settings that has a default value depending on another setting if available
*/
public final String getDefault(Settings settings) {
return defaultValue.apply(settings);
}
/**
* Returns <code>true</code> iff this setting is present in the given settings object. Otherwise <code>false</code>
*/
public final boolean exists(Settings settings) {
return settings.get(key) != null;
}
/**
* Returns the settings value. If the setting is not present in the given settings object the default value is returned
* instead.
*/
public T get(Settings settings) {
String value = getRaw(settings);
try {
return parser.apply(value);
} catch (ElasticsearchParseException ex) {
throw new IllegalArgumentException(ex.getMessage(), ex);
} catch (NumberFormatException ex) {
throw new IllegalArgumentException("Failed to parse value [" + value + "] for setting [" + getKey() + "]", ex);
} catch (IllegalArgumentException ex) {
throw ex;
} catch (Exception t) {
throw new IllegalArgumentException("Failed to parse value [" + value + "] for setting [" + getKey() + "]", t);
}
}
/**
* Returns the raw (string) settings value. If the setting is not present in the given settings object the default value is returned
* instead. This is useful if the value can't be parsed due to an invalid value to access the actual value.
*/
public String getRaw(Settings settings) {
return settings.get(key, defaultValue.apply(settings));
}
/**
* Returns <code>true</code> iff the given key matches the settings key or if this setting is a group setting if the
* given key is part of the settings group.
* @see #isGroupSetting()
*/
public boolean match(String toTest) {
return key.equals(toTest);
}
@Override
public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field("key", key);
builder.field("type", scope.name());
builder.field("dynamic", dynamic);
builder.field("is_group_setting", isGroupSetting());
builder.field("default", defaultValue.apply(Settings.EMPTY));
builder.endObject();
return builder;
}
/**
* The settings scope - settings can either be cluster settings or per index settings.
*/
public enum Scope {
CLUSTER,
INDEX;
}
final AbstractScopedSettings.SettingUpdater newUpdater(Consumer<T> consumer, ESLogger logger) {
return newUpdater(consumer, logger, (s) -> {});
}
AbstractScopedSettings.SettingUpdater newUpdater(Consumer<T> consumer, ESLogger logger, Consumer<T> validator) {
if (isDynamic()) {
return new Updater(consumer, logger, validator);
} else {
throw new IllegalStateException("setting [" + getKey() + "] is not dynamic");
}
}
/**
* this is used for settings that depend on each other... see {@link org.elasticsearch.common.settings.AbstractScopedSettings#addSettingsUpdateConsumer(Setting, Setting, BiConsumer)} and it's
* usage for details.
*/
static <A, B> AbstractScopedSettings.SettingUpdater<Tuple<A, B>> compoundUpdater(final BiConsumer<A,B> consumer, final Setting<A> aSettting, final Setting<B> bSetting, ESLogger logger) {
final AbstractScopedSettings.SettingUpdater<A> aSettingUpdater = aSettting.newUpdater(null, logger);
final AbstractScopedSettings.SettingUpdater<B> bSettingUpdater = bSetting.newUpdater(null, logger);
return new AbstractScopedSettings.SettingUpdater<Tuple<A, B>>() {
@Override
public boolean hasChanged(Settings current, Settings previous) {
return aSettingUpdater.hasChanged(current, previous) || bSettingUpdater.hasChanged(current, previous);
}
@Override
public Tuple<A, B> getValue(Settings current, Settings previous) {
return new Tuple<>(aSettingUpdater.getValue(current, previous), bSettingUpdater.getValue(current, previous));
}
@Override
public void apply(Tuple<A, B> value, Settings current, Settings previous) {
consumer.accept(value.v1(), value.v2());
}
@Override
public String toString() {
return "CompoundUpdater for: " + aSettingUpdater + " and " + bSettingUpdater;
}
};
}
private class Updater implements AbstractScopedSettings.SettingUpdater<T> {
private final Consumer<T> consumer;
private final ESLogger logger;
private final Consumer<T> accept;
public Updater(Consumer<T> consumer, ESLogger logger, Consumer<T> accept) {
this.consumer = consumer;
this.logger = logger;
this.accept = accept;
}
@Override
public String toString() {
return "Updater for: " + Setting.this.toString();
}
@Override
public boolean hasChanged(Settings current, Settings previous) {
final String newValue = getRaw(current);
final String value = getRaw(previous);
assert isGroupSetting() == false : "group settings must override this method";
assert value != null : "value was null but can't be unless default is null which is invalid";
return value.equals(newValue) == false;
}
@Override
public T getValue(Settings current, Settings previous) {
final String newValue = getRaw(current);
final String value = getRaw(previous);
T inst = get(current);
try {
accept.accept(inst);
} catch (Exception | AssertionError e) {
throw new IllegalArgumentException("illegal value can't update [" + key + "] from [" + value + "] to [" + newValue + "]", e);
}
return inst;
}
@Override
public void apply(T value, Settings current, Settings previous) {
logger.info("update [{}] from [{}] to [{}]", key, getRaw(previous), getRaw(current));
consumer.accept(value);
}
}
public Setting(String key, String defaultValue, Function<String, T> parser, boolean dynamic, Scope scope) {
this(key, (s) -> defaultValue, parser, dynamic, scope);
}
public static Setting<Float> floatSetting(String key, float defaultValue, boolean dynamic, Scope scope) {
return new Setting<>(key, (s) -> Float.toString(defaultValue), Float::parseFloat, dynamic, scope);
}
public static Setting<Float> floatSetting(String key, float defaultValue, float minValue, boolean dynamic, Scope scope) {
return new Setting<>(key, (s) -> Float.toString(defaultValue), (s) -> {
float value = Float.parseFloat(s);
if (value < minValue) {
throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be >= " + minValue);
}
return value;
}, dynamic, scope);
}
public static Setting<Integer> intSetting(String key, int defaultValue, int minValue, boolean dynamic, Scope scope) {
return new Setting<>(key, (s) -> Integer.toString(defaultValue), (s) -> parseInt(s, minValue, key), dynamic, scope);
}
public static int parseInt(String s, int minValue, String key) {
int value = Integer.parseInt(s);
if (value < minValue) {
throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be >= " + minValue);
}
return value;
}
public static Setting<Integer> intSetting(String key, int defaultValue, boolean dynamic, Scope scope) {
return intSetting(key, defaultValue, Integer.MIN_VALUE, dynamic, scope);
}
public static Setting<Boolean> boolSetting(String key, boolean defaultValue, boolean dynamic, Scope scope) {
return new Setting<>(key, (s) -> Boolean.toString(defaultValue), Booleans::parseBooleanExact, dynamic, scope);
}
public static Setting<ByteSizeValue> byteSizeSetting(String key, String percentage, boolean dynamic, Scope scope) {
return new Setting<>(key, (s) -> percentage, (s) -> MemorySizeValue.parseBytesSizeValueOrHeapRatio(s, key), dynamic, scope);
}
public static Setting<ByteSizeValue> byteSizeSetting(String key, ByteSizeValue value, boolean dynamic, Scope scope) {
return new Setting<>(key, (s) -> value.toString(), (s) -> ByteSizeValue.parseBytesSizeValue(s, key), dynamic, scope);
}
public static Setting<TimeValue> positiveTimeSetting(String key, TimeValue defaultValue, boolean dynamic, Scope scope) {
return timeSetting(key, defaultValue, TimeValue.timeValueMillis(0), dynamic, scope);
}
public static <T> Setting<List<T>> listSetting(String key, List<String> defaultStringValue, Function<String, T> singleValueParser, boolean dynamic, Scope scope) {
return listSetting(key, (s) -> defaultStringValue, singleValueParser, dynamic, scope);
}
public static <T> Setting<List<T>> listSetting(String key, Function<Settings, List<String>> defaultStringValue, Function<String, T> singleValueParser, boolean dynamic, Scope scope) {
Function<String, List<T>> parser = (s) -> {
try (XContentParser xContentParser = XContentType.JSON.xContent().createParser(s)){
XContentParser.Token token = xContentParser.nextToken();
if (token != XContentParser.Token.START_ARRAY) {
throw new IllegalArgumentException("expected START_ARRAY but got " + token);
}
ArrayList<T> list = new ArrayList<>();
while ((token = xContentParser.nextToken()) !=XContentParser.Token.END_ARRAY) {
if (token != XContentParser.Token.VALUE_STRING) {
throw new IllegalArgumentException("expected VALUE_STRING but got " + token);
}
list.add(singleValueParser.apply(xContentParser.text()));
}
return list;
} catch (IOException e) {
throw new IllegalArgumentException("failed to parse array", e);
}
};
return new Setting<List<T>>(key, (s) -> arrayToParsableString(defaultStringValue.apply(s).toArray(Strings.EMPTY_ARRAY)), parser, dynamic, scope) {
private final Pattern pattern = Pattern.compile(Pattern.quote(key)+"(\\.\\d+)?");
@Override
public String getRaw(Settings settings) {
String[] array = settings.getAsArray(key, null);
return array == null ? defaultValue.apply(settings) : arrayToParsableString(array);
}
public boolean match(String toTest) {
return pattern.matcher(toTest).matches();
}
@Override
boolean hasComplexMatcher() {
return true;
}
};
}
private static String arrayToParsableString(String[] array) {
try {
XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent());
builder.startArray();
for (String element : array) {
builder.value(element);
}
builder.endArray();
return builder.string();
} catch (IOException ex) {
throw new ElasticsearchException(ex);
}
}
public static Setting<Settings> groupSetting(String key, boolean dynamic, Scope scope) {
if (key.endsWith(".") == false) {
throw new IllegalArgumentException("key must end with a '.'");
}
return new Setting<Settings>(key, "", (s) -> null, dynamic, scope) {
@Override
public boolean isGroupSetting() {
return true;
}
@Override
public Settings get(Settings settings) {
return settings.getByPrefix(key);
}
@Override
public boolean match(String toTest) {
return Regex.simpleMatch(key + "*", toTest);
}
@Override
public AbstractScopedSettings.SettingUpdater<Settings> newUpdater(Consumer<Settings> consumer, ESLogger logger, Consumer<Settings> validator) {
if (isDynamic() == false) {
throw new IllegalStateException("setting [" + getKey() + "] is not dynamic");
}
final Setting<?> setting = this;
return new AbstractScopedSettings.SettingUpdater<Settings>() {
@Override
public boolean hasChanged(Settings current, Settings previous) {
Settings currentSettings = get(current);
Settings previousSettings = get(previous);
return currentSettings.equals(previousSettings) == false;
}
@Override
public Settings getValue(Settings current, Settings previous) {
Settings currentSettings = get(current);
Settings previousSettings = get(previous);
try {
validator.accept(currentSettings);
} catch (Exception | AssertionError e) {
throw new IllegalArgumentException("illegal value can't update [" + key + "] from [" + previousSettings.getAsMap() + "] to [" + currentSettings.getAsMap() + "]", e);
}
return currentSettings;
}
@Override
public void apply(Settings value, Settings current, Settings previous) {
consumer.accept(value);
}
@Override
public String toString() {
return "Updater for: " + setting.toString();
}
};
}
};
}
public static Setting<TimeValue> timeSetting(String key, Function<Settings, String> defaultValue, TimeValue minValue, boolean dynamic, Scope scope) {
return new Setting<>(key, defaultValue, (s) -> {
TimeValue timeValue = TimeValue.parseTimeValue(s, null, key);
if (timeValue.millis() < minValue.millis()) {
throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be >= " + minValue);
}
return timeValue;
}, dynamic, scope);
}
public static Setting<TimeValue> timeSetting(String key, TimeValue defaultValue, TimeValue minValue, boolean dynamic, Scope scope) {
return timeSetting(key, (s) -> defaultValue.getStringRep(), minValue, dynamic, scope);
}
public static Setting<TimeValue> timeSetting(String key, TimeValue defaultValue, boolean dynamic, Scope scope) {
return new Setting<>(key, (s) -> defaultValue.toString(), (s) -> TimeValue.parseTimeValue(s, defaultValue, key), dynamic, scope);
}
public static Setting<Double> doubleSetting(String key, double defaultValue, double minValue, boolean dynamic, Scope scope) {
return new Setting<>(key, (s) -> Double.toString(defaultValue), (s) -> {
final double d = Double.parseDouble(s);
if (d < minValue) {
throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be >= " + minValue);
}
return d;
}, dynamic, scope);
}
}

23
core/src/main/java/org/elasticsearch/common/settings/Settings.java
View File

@ -597,6 +597,8 @@ public final class Settings implements ToXContent {
return result.toArray(new String[result.size()]);
}
/**
* Returns group settings for the given setting prefix.
*/
@ -614,6 +616,9 @@ public final class Settings implements ToXContent {
if (settingPrefix.charAt(settingPrefix.length() - 1) != '.') {
settingPrefix = settingPrefix + ".";
}
return getGroupsInternal(settingPrefix, ignoreNonGrouped);
}
private Map<String, Settings> getGroupsInternal(String settingPrefix, boolean ignoreNonGrouped) throws SettingsException {
// we don't really care that it might happen twice
Map<String, Map<String, String>> map = new LinkedHashMap<>();
for (Object o : settings.keySet()) {
@ -643,6 +648,16 @@ public final class Settings implements ToXContent {
}
return Collections.unmodifiableMap(retVal);
}
/**
* Returns group settings for the given setting prefix.
*/
public Map<String, Settings> getAsGroups() throws SettingsException {
return getAsGroups(false);
}
public Map<String, Settings> getAsGroups(boolean ignoreNonGrouped) throws SettingsException {
return getGroupsInternal("", ignoreNonGrouped);
}
/**
* Returns a parsed version.
@ -706,7 +721,7 @@ public final class Settings implements ToXContent {
Builder builder = new Builder();
int numberOfSettings = in.readVInt();
for (int i = 0; i < numberOfSettings; i++) {
builder.put(in.readString(), in.readString());
builder.put(in.readString(), in.readOptionalString());
}
return builder.build();
}
@ -715,7 +730,7 @@ public final class Settings implements ToXContent {
out.writeVInt(settings.getAsMap().size());
for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
out.writeString(entry.getKey());
out.writeString(entry.getValue());
out.writeOptionalString(entry.getValue());
}
}
@ -818,6 +833,10 @@ public final class Settings implements ToXContent {
return this;
}
public Builder putNull(String key) {
return put(key, (String) null);
}
/**
* Sets a setting with the provided setting key and class as value.
*

27
core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
View File

@ -21,6 +21,10 @@ package org.elasticsearch.common.settings;
import org.elasticsearch.common.inject.AbstractModule;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
/**
* A module that binds the provided settings to the {@link Settings} interface.
*
@ -30,15 +34,36 @@ public class SettingsModule extends AbstractModule {
private final Settings settings;
private final SettingsFilter settingsFilter;
private final Map<String, Setting<?>> clusterDynamicSettings = new HashMap<>();
public SettingsModule(Settings settings, SettingsFilter settingsFilter) {
this.settings = settings;
this.settingsFilter = settingsFilter;
for (Setting<?> setting : ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) {
registerSetting(setting);
}
}
@Override
protected void configure() {
bind(Settings.class).toInstance(settings);
bind(SettingsFilter.class).toInstance(settingsFilter);
final ClusterSettings clusterSettings = new ClusterSettings(settings, new HashSet<>(clusterDynamicSettings.values()));
bind(ClusterSettings.class).toInstance(clusterSettings);
}
}
public void registerSetting(Setting<?> setting) {
switch (setting.getScope()) {
case CLUSTER:
if (clusterDynamicSettings.containsKey(setting.getKey())) {
throw new IllegalArgumentException("Cannot register setting [" + setting.getKey() + "] twice");
}
clusterDynamicSettings.put(setting.getKey(), setting);
break;
case INDEX:
throw new UnsupportedOperationException("not yet implemented");
}
}
}

18
core/src/main/java/org/elasticsearch/common/settings/loader/XContentSettingsLoader.java
View File

@ -103,9 +103,9 @@ public abstract class XContentSettingsLoader implements SettingsLoader {
} else if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.VALUE_NULL) {
// ignore this
serializeValue(settings, sb, path, parser, currentFieldName, true);
} else {
serializeValue(settings, sb, path, parser, currentFieldName);
serializeValue(settings, sb, path, parser, currentFieldName, false);
}
}
@ -126,31 +126,33 @@ public abstract class XContentSettingsLoader implements SettingsLoader {
} else if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token == XContentParser.Token.VALUE_NULL) {
serializeValue(settings, sb, path, parser, fieldName + '.' + (counter++), true);
// ignore
} else {
serializeValue(settings, sb, path, parser, fieldName + '.' + (counter++));
serializeValue(settings, sb, path, parser, fieldName + '.' + (counter++), false);
}
}
}
private void serializeValue(Map<String, String> settings, StringBuilder sb, List<String> path, XContentParser parser, String fieldName) throws IOException {
private void serializeValue(Map<String, String> settings, StringBuilder sb, List<String> path, XContentParser parser, String fieldName, boolean isNull) throws IOException {
sb.setLength(0);
for (String pathEle : path) {
sb.append(pathEle).append('.');
}
sb.append(fieldName);
String key = sb.toString();
String currentValue = parser.text();
String previousValue = settings.put(key, currentValue);
if (previousValue != null) {
String currentValue = isNull ? null : parser.text();
if (settings.containsKey(key)) {
throw new ElasticsearchParseException(
"duplicate settings key [{}] found at line number [{}], column number [{}], previous value [{}], current value [{}]",
key,
parser.getTokenLocation().lineNumber,
parser.getTokenLocation().columnNumber,
previousValue,
settings.get(key),
currentValue
);
}
settings.put(key, currentValue);
}
}

82
core/src/main/java/org/elasticsearch/common/text/BytesText.java
View File

@ -1,82 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.text;
import java.nio.charset.StandardCharsets;
import org.elasticsearch.common.bytes.BytesReference;
/**
* A {@link BytesReference} representation of the text, will always convert on the fly to a {@link String}.
*/
public class BytesText implements Text {
private BytesReference bytes;
private int hash;
public BytesText(BytesReference bytes) {
this.bytes = bytes;
}
@Override
public boolean hasBytes() {
return true;
}
@Override
public BytesReference bytes() {
return bytes;
}
@Override
public boolean hasString() {
return false;
}
@Override
public String string() {
// TODO: we can optimize the conversion based on the bytes reference API similar to UnicodeUtil
if (!bytes.hasArray()) {
bytes = bytes.toBytesArray();
}
return new String(bytes.array(), bytes.arrayOffset(), bytes.length(), StandardCharsets.UTF_8);
}
@Override
public String toString() {
return string();
}
@Override
public int hashCode() {
if (hash == 0) {
hash = bytes.hashCode();
}
return hash;
}
@Override
public boolean equals(Object obj) {
return bytes().equals(((Text) obj).bytes());
}
@Override
public int compareTo(Text text) {
return UTF8SortedAsUnicodeComparator.utf8SortedAsUnicodeSortOrder.compare(bytes(), text.bytes());
}
}

111
core/src/main/java/org/elasticsearch/common/text/StringAndBytesText.java
View File

@ -1,111 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.text;
import java.nio.charset.StandardCharsets;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
/**
* Both {@link String} and {@link BytesReference} representation of the text. Starts with one of those, and if
* the other is requests, caches the other one in a local reference so no additional conversion will be needed.
*/
public class StringAndBytesText implements Text {
public static final Text[] EMPTY_ARRAY = new Text[0];
public static Text[] convertFromStringArray(String[] strings) {
if (strings.length == 0) {
return EMPTY_ARRAY;
}
Text[] texts = new Text[strings.length];
for (int i = 0; i < strings.length; i++) {
texts[i] = new StringAndBytesText(strings[i]);
}
return texts;
}
private BytesReference bytes;
private String text;
private int hash;
public StringAndBytesText(BytesReference bytes) {
this.bytes = bytes;
}
public StringAndBytesText(String text) {
this.text = text;
}
@Override
public boolean hasBytes() {
return bytes != null;
}
@Override
public BytesReference bytes() {
if (bytes == null) {
bytes = new BytesArray(text.getBytes(StandardCharsets.UTF_8));
}
return bytes;
}
@Override
public boolean hasString() {
return text != null;
}
@Override
public String string() {
// TODO: we can optimize the conversion based on the bytes reference API similar to UnicodeUtil
if (text == null) {
if (!bytes.hasArray()) {
bytes = bytes.toBytesArray();
}
text = new String(bytes.array(), bytes.arrayOffset(), bytes.length(), StandardCharsets.UTF_8);
}
return text;
}
@Override
public String toString() {
return string();
}
@Override
public int hashCode() {
if (hash == 0) {
hash = bytes().hashCode();
}
return hash;
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
return bytes().equals(((Text) obj).bytes());
}
@Override
public int compareTo(Text text) {
return UTF8SortedAsUnicodeComparator.utf8SortedAsUnicodeSortOrder.compare(bytes(), text.bytes());
}
}

Some files were not shown because too many files have changed in this diff Show More